A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Model] Adding Support for Qwen2VL as an Embedding Model. Using MrLight/dse-qwen2-2b-mrl-v1 (#9944)
Signed-off-by: FurtherAI <austin.veselka@lighton.ai> Co-authored-by: FurtherAI <austin.veselka@lighton.ai>
Austin Veselka committed
Nov 13, 2024 at 02:28 UTC
1b886aa104248a95720fda7be9f979fc665b3d02
8 files changed
+364
-19
docs/source/models/supported_models.rst
+6
@@ -584,6 +584,12 @@ Multimodal Embedding
584
- :code:`TIGER-Lab/VLM2Vec-Full`
585
- 🚧
586
- ✅︎
587
+ * - :code:`Qwen2VLForConditionalGeneration`
588
+ - Qwen2-VL-based
589
+ - T + I
590
+ - :code:`MrLight/dse-qwen2-2b-mrl-v1`
591
+ -
592
+ - ✅︎
593
594
.. important::
595
Some model architectures support both generation and embedding tasks.
docs/source/models/vlm.rst
+17
@@ -310,4 +310,21 @@ Since the request schema is not defined by OpenAI client, we post a request to t
310
response_json = response.json()
311
print("Embedding output:", response_json["data"][0]["embedding"])
312
313
+Here is an example for serving the ``MrLight/dse-qwen2-2b-mrl-v1`` model.
314
+
315
+.. code-block:: bash
316
+
317
+ vllm serve MrLight/dse-qwen2-2b-mrl-v1 --task embedding \
318
+ --trust-remote-code --max-model-len 8192 --chat-template examples/template_dse_qwen2_vl.jinja
319
+
320
+.. important::
321
+
322
+ Like with VLM2Vec, we have to explicitly pass ``--task embedding``. Additionally, ``MrLight/dse-qwen2-2b-mrl-v1`` requires an EOS token for embeddings,
323
+ which is handled by the jinja template.
324
+
325
+.. important::
326
+
327
+ Also important, ``MrLight/dse-qwen2-2b-mrl-v1`` requires a placeholder image of the minimum image size for text query embeddings. See the full code
328
+ example below for details.
329
+
330
A full code example can be found in `examples/openai_chat_embedding_client_for_multimodal.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_embedding_client_for_multimodal.py>`_.
examples/openai_chat_embedding_client_for_multimodal.py
+105
-18
@@ -1,33 +1,120 @@
1
+import argparse
2
+import base64
3
+import io
4
+
5
import requests
6
+from PIL import Image
7
8
image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
9
5
-response = requests.post(
6
- "http://localhost:8000/v1/embeddings",
7
- json={
8
- "model":
9
- "TIGER-Lab/VLM2Vec-Full",
10
- "messages": [{
10
+
11
+def vlm2vec():
12
+ response = requests.post(
13
+ "http://localhost:8000/v1/embeddings",
14
+ json={
15
+ "model":
16
+ "TIGER-Lab/VLM2Vec-Full",
17
+ "messages": [{
18
+ "role":
19
+ "user",
20
+ "content": [
21
+ {
22
+ "type": "image_url",
23
+ "image_url": {
24
+ "url": image_url
25
+ }
26
+ },
27
+ {
28
+ "type": "text",
29
+ "text": "Represent the given image."
30
+ },
31
+ ],
32
+ }],
33
+ "encoding_format":
34
+ "float",
35
+ },
36
+ )
37
+ response.raise_for_status()
38
+ response_json = response.json()
39
+
40
+ print("Embedding output:", response_json["data"][0]["embedding"])
41
+
42
+
43
+def dse_qwen2_vl(inp: dict):
44
+ # Embedding an Image
45
+ if inp["dtype"] == "image":
46
+ messages = [{
47
+ "role":
48
+ "user",
49
+ "content": [{
50
+ "type": "image_url",
51
+ "image_url": {
52
+ "url": inp["image_url"],
53
+ }
54
+ }, {
55
+ "type": "text",
56
+ "text": "What is shown in this image?"
57
+ }]
58
+ }]
59
+ # Embedding a Text Query
60
+ else:
61
+ # MrLight/dse-qwen2-2b-mrl-v1 requires a placeholder image
62
+ # of the minimum input size
63
+ buffer = io.BytesIO()
64
+ image_placeholder = Image.new("RGB", (56, 56))
65
+ image_placeholder.save(buffer, "png")
66
+ buffer.seek(0)
67
+ image_placeholder = base64.b64encode(buffer.read()).decode('utf-8')
68
+ messages = [{
69
"role":
70
"user",
71
"content": [
72
{
73
"type": "image_url",
74
"image_url": {
17
- "url": image_url
75
+ "url": f"data:image/jpeg;base64,{image_placeholder}",
76
}
77
},
78
{
79
"type": "text",
22
- "text": "Represent the given image."
80
+ "text": f"Query: {inp['content']}"
81
},
24
- ],
25
- }],
26
- "encoding_format":
27
- "float",
28
- },
29
-)
30
-response.raise_for_status()
31
-response_json = response.json()
32
-
33
-print("Embedding output:", response_json["data"][0]["embedding"])
82
+ ]
83
+ }]
84
+
85
+ response = requests.post(
86
+ "http://localhost:8000/v1/embeddings",
87
+ json={
88
+ "model": "MrLight/dse-qwen2-2b-mrl-v1",
89
+ "messages": messages,
90
+ "encoding_format": "float",
91
+ },
92
+ )
93
+ response.raise_for_status()
94
+ response_json = response.json()
95
+
96
+ print("Embedding output:", response_json["data"][0]["embedding"])
97
+
98
+
99
+if __name__ == '__main__':
100
+ parser = argparse.ArgumentParser(
101
+ "Script to call a specified VLM through the API. Make sure to serve "
102
+ "the model with --task embedding before running this.")
103
+ parser.add_argument("model",
104
+ type=str,
105
+ choices=["vlm2vec", "dse_qwen2_vl"],
106
+ required=True,
107
+ help="Which model to call.")
108
+ args = parser.parse_args()
109
+
110
+ if args.model == "vlm2vec":
111
+ vlm2vec()
112
+ elif args.model == "dse_qwen2_vl":
113
+ dse_qwen2_vl({
114
+ "dtye": "image",
115
+ "image_url": image_url,
116
+ })
117
+ dse_qwen2_vl({
118
+ "dtype": "text",
119
+ "content": "What is the weather like today?",
120
+ })
examples/template_dse_qwen2_vl.jinja
new
+7
@@ -0,0 +1,7 @@
1
+{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}{% raw %}<|im_start|>system
2
+You are a helpful assistant.<|im_end|>
3
+{% endraw %}{% endif %}<|im_start|>{{ message['role'] }}{% raw %}
4
+{% endraw %}{% if message['content'] is string %}{{ message['content'] }}<|im_end|>{% raw %}
5
+{% endraw %}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>{% raw %}
6
+{% endraw %}{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant{% raw %}
7
+{% endraw %}{% endif %}<|endoftext|>
\ No newline at end of file
tests/conftest.py
+3
@@ -243,6 +243,9 @@ _T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)
243
class HfRunner:
244
245
def wrap_device(self, x: _T, device: Optional[str] = None) -> _T:
246
+ if x is None or isinstance(x, (bool, )):
247
+ return x
248
+
249
if device is None:
250
device = "cpu" if current_platform.is_cpu() else "cuda"
251
tests/models/embedding/vision_language/test_dse_qwen2_vl.py
new
+209
@@ -0,0 +1,209 @@
1
+from functools import partial
2
+from typing import Callable, Dict, List, Type
3
+
4
+import pytest
5
+import torch
6
+from PIL import Image
7
+from transformers import BatchEncoding, Qwen2VLForConditionalGeneration
8
+
9
+from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
10
+from ....utils import large_gpu_test
11
+from ..utils import check_embeddings_close
12
+
13
+HF_TEXT_PROMPTS = [
14
+ # T -> X
15
+ (
16
+ "Query: Find me an everyday image that matches the given caption: The label of the object is stop sign", # noqa: E501,
17
+ Image.new("RGB", (56, 56))),
18
+ # T -> X
19
+ ("Query: Retrieve an image of this caption: cherry blossom",
20
+ Image.new("RGB", (56, 56))),
21
+]
22
+
23
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
24
+ "stop_sign":
25
+ "What is shown in this image?",
26
+ "cherry_blossom":
27
+ "What is shown in this image?"
28
+})
29
+
30
+MODELS = ["MrLight/dse-qwen2-2b-mrl-v1"]
31
+
32
+
33
+def get_messages(image: Image.Image, text: str, embed_text: bool):
34
+ # assert False, 'remember to use outer [] as required'
35
+ if embed_text:
36
+ messages = [{
37
+ "role":
38
+ "user",
39
+ "content": [
40
+ {
41
+ "type": "image",
42
+ "image": Image.new("RGB", (56, 56)),
43
+ "resized_height": 1,
44
+ "resized_width": 1
45
+ }, # need a dummy image here for an easier process.
46
+ {
47
+ "type": "text",
48
+ "text": text
49
+ },
50
+ ]
51
+ }]
52
+ else:
53
+ messages = [{
54
+ "role":
55
+ "user",
56
+ "content": [{
57
+ "type": "image",
58
+ "image": image
59
+ }, {
60
+ "type": "text",
61
+ "text": text
62
+ }]
63
+ }]
64
+ return messages
65
+
66
+
67
+def apply_chat_template_and_add_eos(
68
+ messages: List[Dict],
69
+ apply_chat_template_fn: Callable,
70
+):
71
+ prompt = apply_chat_template_fn(
72
+ messages, tokenize=False, add_generation_prompt=True) + "<|endoftext|>"
73
+ return prompt
74
+
75
+
76
+def postprocess_inputs(hf_model: HfRunner, inputs: BatchEncoding, **kwargs):
77
+ return hf_model.model.prepare_inputs_for_generation(**inputs, **kwargs)
78
+
79
+
80
+def _run_test(
81
+ hf_runner: Type[HfRunner],
82
+ vllm_runner: Type[VllmRunner],
83
+ input_texts: List[str],
84
+ input_images: PromptImageInput,
85
+ embed_texts: List[bool],
86
+ model: str,
87
+ *,
88
+ dtype: str,
89
+) -> None:
90
+ '''SET PYTHONPATH'''
91
+ # NOTE: take care of the order. run vLLM first, and then run HF.
92
+ # vLLM needs a fresh new process without cuda initialization.
93
+ # if we run HF first, the cuda initialization will be done and it
94
+ # will hurt multiprocessing backend with fork method (the default method).
95
+ with vllm_runner(model,
96
+ task="embedding",
97
+ dtype=dtype,
98
+ enforce_eager=True,
99
+ max_model_len=8192) as vllm_model:
100
+ tokenizer = vllm_model.model.get_tokenizer()
101
+ texts = [
102
+ # this is necessary because vllm_model.encode will not apply any
103
+ # templating to the prompt, and therefore lacks an image_pad
104
+ # token unless one is inserted beforehand (the (28,28) image
105
+ # above is converted to an image pad token by the chat template).
106
+ apply_chat_template_and_add_eos(
107
+ get_messages(image, text, False),
108
+ apply_chat_template_fn=tokenizer.apply_chat_template,
109
+ ) for text, image in zip(input_texts, input_images)
110
+ # vllm will replace the pad token with the actual image,
111
+ # which may be a placeholder image, later.
112
+ ]
113
+ vllm_outputs = vllm_model.encode(texts, images=input_images)
114
+
115
+ hf_outputs = []
116
+ with hf_runner(model,
117
+ dtype=dtype,
118
+ auto_cls=Qwen2VLForConditionalGeneration) as hf_model:
119
+ hf_model.postprocess_inputs = partial(
120
+ postprocess_inputs,
121
+ hf_model,
122
+ cache_position=torch.arange(
123
+ 0,
124
+ 1, # 1 for batch size
125
+ requires_grad=False),
126
+ use_cache=False)
127
+ for text, image, embed_text in zip(input_texts, input_images,
128
+ embed_texts):
129
+ # dse requires non-standard input processing
130
+ # because it needs an image_pad token
131
+ messages = get_messages(image, text, embed_text)
132
+ prompt = apply_chat_template_and_add_eos(
133
+ messages, hf_model.processor.apply_chat_template)
134
+ inputs = hf_model.get_inputs(
135
+ prompts=[[prompt]],
136
+ images=[[image]],
137
+ )
138
+ with torch.no_grad():
139
+ outputs = hf_model.model(
140
+ **hf_model.wrap_device(inputs[0],
141
+ device=hf_model.model.device.type),
142
+ return_dict=True,
143
+ output_hidden_states=True,
144
+ )
145
+ pooled_output = torch.nn.functional.normalize(
146
+ outputs.hidden_states[-1][0, -1], p=2, dim=-1)
147
+ hf_outputs.append(pooled_output.tolist())
148
+
149
+ check_embeddings_close(
150
+ embeddings_0_lst=hf_outputs,
151
+ embeddings_1_lst=vllm_outputs,
152
+ name_0="hf",
153
+ name_1="vllm",
154
+ )
155
+
156
+
157
+@pytest.mark.parametrize("model", MODELS)
158
+@pytest.mark.parametrize("dtype", ["bfloat16"])
159
+def test_models_text(
160
+ hf_runner,
161
+ vllm_runner,
162
+ image_assets,
163
+ model: str,
164
+ dtype: str,
165
+) -> None:
166
+ input_texts_images = [(text, image_placeholder)
167
+ for text, image_placeholder in HF_TEXT_PROMPTS]
168
+ input_texts = [text for text, _ in input_texts_images]
169
+ input_images = [image for _, image in input_texts_images]
170
+ embed_texts = [True] * len(input_texts)
171
+
172
+ _run_test(
173
+ hf_runner,
174
+ vllm_runner,
175
+ input_texts,
176
+ input_images, # type: ignore
177
+ embed_texts,
178
+ model,
179
+ dtype=dtype,
180
+ )
181
+
182
+
183
+@large_gpu_test(min_gb=48)
184
+@pytest.mark.parametrize("model", MODELS)
185
+@pytest.mark.parametrize("dtype", ["bfloat16"])
186
+def test_models_image(
187
+ hf_runner,
188
+ vllm_runner,
189
+ image_assets,
190
+ model: str,
191
+ dtype: str,
192
+) -> None:
193
+ input_texts_images = [
194
+ (text, asset.pil_image)
195
+ for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
196
+ ]
197
+ input_texts = [text for text, _ in input_texts_images]
198
+ input_images = [image for _, image in input_texts_images]
199
+ embed_texts = [False] * len(input_texts)
200
+
201
+ _run_test(
202
+ hf_runner,
203
+ vllm_runner,
204
+ input_texts,
205
+ input_images,
206
+ embed_texts,
207
+ model,
208
+ dtype=dtype,
209
+ )
vllm/model_executor/models/qwen2_vl.py
+16
-1
@@ -51,6 +51,7 @@ from vllm.model_executor.layers.activation import QuickGELU
51
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
52
RowParallelLinear)
53
from vllm.model_executor.layers.logits_processor import LogitsProcessor
54
+from vllm.model_executor.layers.pooler import Pooler, PoolingType
55
from vllm.model_executor.layers.quantization import (GPTQConfig,
56
GPTQMarlinConfig,
57
QuantizationConfig)
@@ -58,12 +59,13 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
59
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
60
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
61
from vllm.model_executor.models.qwen2 import Qwen2Model
62
+from vllm.model_executor.pooling_metadata import PoolingMetadata
63
from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
64
MultiModalKwargs)
65
from vllm.multimodal.base import MultiModalData
66
from vllm.multimodal.image import cached_get_image_processor
67
from vllm.multimodal.utils import cached_get_tokenizer
66
-from vllm.sequence import IntermediateTensors, SequenceData
68
+from vllm.sequence import IntermediateTensors, PoolerOutput, SequenceData
69
from vllm.transformers_utils.config import uses_mrope
70
from vllm.transformers_utils.processor import cached_get_processor
71
@@ -1067,6 +1069,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1069
config = vllm_config.model_config.hf_config
1070
cache_config = vllm_config.cache_config
1071
quant_config = vllm_config.quant_config
1072
+ pooler_config = vllm_config.model_config.pooler_config
1073
multimodal_config = vllm_config.model_config.multimodal_config
1074
assert not cache_config.enable_prefix_caching, \
1075
"Qwen2-VL currently does not support prefix caching"
@@ -1098,6 +1101,11 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1101
1102
self.logits_processor = LogitsProcessor(config.vocab_size)
1103
self.sampler = get_sampler()
1104
+ self._pooler = Pooler.from_config_with_defaults(
1105
+ pooler_config,
1106
+ pooling_type=PoolingType.LAST,
1107
+ normalize=True,
1108
+ softmax=False)
1109
self.make_empty_intermediate_tensors = (
1110
make_empty_intermediate_tensors_factory(
1111
["hidden_states", "residual"], config.hidden_size))
@@ -1318,6 +1326,13 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1326
next_tokens = self.sampler(logits, sampling_metadata)
1327
return next_tokens
1328
1329
+ def pooler(
1330
+ self,
1331
+ hidden_states: torch.Tensor,
1332
+ pooling_metadata: PoolingMetadata,
1333
+ ) -> Optional[PoolerOutput]:
1334
+ return self._pooler(hidden_states, pooling_metadata)
1335
+
1336
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1337
stacked_params_mapping = [
1338
# (param_name, shard_name, shard_id)
vllm/model_executor/models/registry.py
+1
@@ -109,6 +109,7 @@ _EMBEDDING_MODELS = {
109
# [Multimodal]
110
"LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
111
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
112
+ "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration") # noqa: E501,
113
}
114
115
_MULTIMODAL_MODELS = {