@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1b886aa104
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Model] Adding Support for Qwen2VL as an Embedding Model. Using MrLight/dse-qwen2-2b-mrl-v1 (#9944)

Signed-off-by: FurtherAI <austin.veselka@lighton.ai> Co-authored-by: FurtherAI <austin.veselka@lighton.ai>

Austin Veselka committed Nov 13, 2024 at 02:28 UTC 1b886aa104248a95720fda7be9f979fc665b3d02
8 files changed +364 -19
docs/source/models/supported_models.rst
+6
@@ -584,6 +584,12 @@ Multimodal Embedding
584 - :code:`TIGER-Lab/VLM2Vec-Full`
585 - 🚧
586 - ✅︎
587 + * - :code:`Qwen2VLForConditionalGeneration`
588 + - Qwen2-VL-based
589 + - T + I
590 + - :code:`MrLight/dse-qwen2-2b-mrl-v1`
591 + -
592 + - ✅︎
593
594 .. important::
595 Some model architectures support both generation and embedding tasks.
docs/source/models/vlm.rst
+17
@@ -310,4 +310,21 @@ Since the request schema is not defined by OpenAI client, we post a request to t
310 response_json = response.json()
311 print("Embedding output:", response_json["data"][0]["embedding"])
312
313 +Here is an example for serving the ``MrLight/dse-qwen2-2b-mrl-v1`` model.
314 +
315 +.. code-block:: bash
316 +
317 + vllm serve MrLight/dse-qwen2-2b-mrl-v1 --task embedding \
318 + --trust-remote-code --max-model-len 8192 --chat-template examples/template_dse_qwen2_vl.jinja
319 +
320 +.. important::
321 +
322 + Like with VLM2Vec, we have to explicitly pass ``--task embedding``. Additionally, ``MrLight/dse-qwen2-2b-mrl-v1`` requires an EOS token for embeddings,
323 + which is handled by the jinja template.
324 +
325 +.. important::
326 +
327 + Also important, ``MrLight/dse-qwen2-2b-mrl-v1`` requires a placeholder image of the minimum image size for text query embeddings. See the full code
328 + example below for details.
329 +
330 A full code example can be found in `examples/openai_chat_embedding_client_for_multimodal.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_embedding_client_for_multimodal.py>`_.
examples/openai_chat_embedding_client_for_multimodal.py
+105 -18
@@ -1,33 +1,120 @@
1 +import argparse
2 +import base64
3 +import io
4 +
5 import requests
6 +from PIL import Image
7
8 image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
9
5 -response = requests.post(
6 - "http://localhost:8000/v1/embeddings",
7 - json={
8 - "model":
9 - "TIGER-Lab/VLM2Vec-Full",
10 - "messages": [{
10 +
11 +def vlm2vec():
12 + response = requests.post(
13 + "http://localhost:8000/v1/embeddings",
14 + json={
15 + "model":
16 + "TIGER-Lab/VLM2Vec-Full",
17 + "messages": [{
18 + "role":
19 + "user",
20 + "content": [
21 + {
22 + "type": "image_url",
23 + "image_url": {
24 + "url": image_url
25 + }
26 + },
27 + {
28 + "type": "text",
29 + "text": "Represent the given image."
30 + },
31 + ],
32 + }],
33 + "encoding_format":
34 + "float",
35 + },
36 + )
37 + response.raise_for_status()
38 + response_json = response.json()
39 +
40 + print("Embedding output:", response_json["data"][0]["embedding"])
41 +
42 +
43 +def dse_qwen2_vl(inp: dict):
44 + # Embedding an Image
45 + if inp["dtype"] == "image":
46 + messages = [{
47 + "role":
48 + "user",
49 + "content": [{
50 + "type": "image_url",
51 + "image_url": {
52 + "url": inp["image_url"],
53 + }
54 + }, {
55 + "type": "text",
56 + "text": "What is shown in this image?"
57 + }]
58 + }]
59 + # Embedding a Text Query
60 + else:
61 + # MrLight/dse-qwen2-2b-mrl-v1 requires a placeholder image
62 + # of the minimum input size
63 + buffer = io.BytesIO()
64 + image_placeholder = Image.new("RGB", (56, 56))
65 + image_placeholder.save(buffer, "png")
66 + buffer.seek(0)
67 + image_placeholder = base64.b64encode(buffer.read()).decode('utf-8')
68 + messages = [{
69 "role":
70 "user",
71 "content": [
72 {
73 "type": "image_url",
74 "image_url": {
17 - "url": image_url
75 + "url": f"data:image/jpeg;base64,{image_placeholder}",
76 }
77 },
78 {
79 "type": "text",
22 - "text": "Represent the given image."
80 + "text": f"Query: {inp['content']}"
81 },
24 - ],
25 - }],
26 - "encoding_format":
27 - "float",
28 - },
29 -)
30 -response.raise_for_status()
31 -response_json = response.json()
32 -
33 -print("Embedding output:", response_json["data"][0]["embedding"])
82 + ]
83 + }]
84 +
85 + response = requests.post(
86 + "http://localhost:8000/v1/embeddings",
87 + json={
88 + "model": "MrLight/dse-qwen2-2b-mrl-v1",
89 + "messages": messages,
90 + "encoding_format": "float",
91 + },
92 + )
93 + response.raise_for_status()
94 + response_json = response.json()
95 +
96 + print("Embedding output:", response_json["data"][0]["embedding"])
97 +
98 +
99 +if __name__ == '__main__':
100 + parser = argparse.ArgumentParser(
101 + "Script to call a specified VLM through the API. Make sure to serve "
102 + "the model with --task embedding before running this.")
103 + parser.add_argument("model",
104 + type=str,
105 + choices=["vlm2vec", "dse_qwen2_vl"],
106 + required=True,
107 + help="Which model to call.")
108 + args = parser.parse_args()
109 +
110 + if args.model == "vlm2vec":
111 + vlm2vec()
112 + elif args.model == "dse_qwen2_vl":
113 + dse_qwen2_vl({
114 + "dtye": "image",
115 + "image_url": image_url,
116 + })
117 + dse_qwen2_vl({
118 + "dtype": "text",
119 + "content": "What is the weather like today?",
120 + })
examples/template_dse_qwen2_vl.jinja new
+7
@@ -0,0 +1,7 @@
1 +{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}{% raw %}<|im_start|>system
2 +You are a helpful assistant.<|im_end|>
3 +{% endraw %}{% endif %}<|im_start|>{{ message['role'] }}{% raw %}
4 +{% endraw %}{% if message['content'] is string %}{{ message['content'] }}<|im_end|>{% raw %}
5 +{% endraw %}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>{% raw %}
6 +{% endraw %}{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant{% raw %}
7 +{% endraw %}{% endif %}<|endoftext|>
\ No newline at end of file
tests/conftest.py
+3
@@ -243,6 +243,9 @@ _T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)
243 class HfRunner:
244
245 def wrap_device(self, x: _T, device: Optional[str] = None) -> _T:
246 + if x is None or isinstance(x, (bool, )):
247 + return x
248 +
249 if device is None:
250 device = "cpu" if current_platform.is_cpu() else "cuda"
251
tests/models/embedding/vision_language/test_dse_qwen2_vl.py new
+209
@@ -0,0 +1,209 @@
1 +from functools import partial
2 +from typing import Callable, Dict, List, Type
3 +
4 +import pytest
5 +import torch
6 +from PIL import Image
7 +from transformers import BatchEncoding, Qwen2VLForConditionalGeneration
8 +
9 +from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
10 +from ....utils import large_gpu_test
11 +from ..utils import check_embeddings_close
12 +
13 +HF_TEXT_PROMPTS = [
14 + # T -> X
15 + (
16 + "Query: Find me an everyday image that matches the given caption: The label of the object is stop sign", # noqa: E501,
17 + Image.new("RGB", (56, 56))),
18 + # T -> X
19 + ("Query: Retrieve an image of this caption: cherry blossom",
20 + Image.new("RGB", (56, 56))),
21 +]
22 +
23 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
24 + "stop_sign":
25 + "What is shown in this image?",
26 + "cherry_blossom":
27 + "What is shown in this image?"
28 +})
29 +
30 +MODELS = ["MrLight/dse-qwen2-2b-mrl-v1"]
31 +
32 +
33 +def get_messages(image: Image.Image, text: str, embed_text: bool):
34 + # assert False, 'remember to use outer [] as required'
35 + if embed_text:
36 + messages = [{
37 + "role":
38 + "user",
39 + "content": [
40 + {
41 + "type": "image",
42 + "image": Image.new("RGB", (56, 56)),
43 + "resized_height": 1,
44 + "resized_width": 1
45 + }, # need a dummy image here for an easier process.
46 + {
47 + "type": "text",
48 + "text": text
49 + },
50 + ]
51 + }]
52 + else:
53 + messages = [{
54 + "role":
55 + "user",
56 + "content": [{
57 + "type": "image",
58 + "image": image
59 + }, {
60 + "type": "text",
61 + "text": text
62 + }]
63 + }]
64 + return messages
65 +
66 +
67 +def apply_chat_template_and_add_eos(
68 + messages: List[Dict],
69 + apply_chat_template_fn: Callable,
70 +):
71 + prompt = apply_chat_template_fn(
72 + messages, tokenize=False, add_generation_prompt=True) + "<|endoftext|>"
73 + return prompt
74 +
75 +
76 +def postprocess_inputs(hf_model: HfRunner, inputs: BatchEncoding, **kwargs):
77 + return hf_model.model.prepare_inputs_for_generation(**inputs, **kwargs)
78 +
79 +
80 +def _run_test(
81 + hf_runner: Type[HfRunner],
82 + vllm_runner: Type[VllmRunner],
83 + input_texts: List[str],
84 + input_images: PromptImageInput,
85 + embed_texts: List[bool],
86 + model: str,
87 + *,
88 + dtype: str,
89 +) -> None:
90 + '''SET PYTHONPATH'''
91 + # NOTE: take care of the order. run vLLM first, and then run HF.
92 + # vLLM needs a fresh new process without cuda initialization.
93 + # if we run HF first, the cuda initialization will be done and it
94 + # will hurt multiprocessing backend with fork method (the default method).
95 + with vllm_runner(model,
96 + task="embedding",
97 + dtype=dtype,
98 + enforce_eager=True,
99 + max_model_len=8192) as vllm_model:
100 + tokenizer = vllm_model.model.get_tokenizer()
101 + texts = [
102 + # this is necessary because vllm_model.encode will not apply any
103 + # templating to the prompt, and therefore lacks an image_pad
104 + # token unless one is inserted beforehand (the (28,28) image
105 + # above is converted to an image pad token by the chat template).
106 + apply_chat_template_and_add_eos(
107 + get_messages(image, text, False),
108 + apply_chat_template_fn=tokenizer.apply_chat_template,
109 + ) for text, image in zip(input_texts, input_images)
110 + # vllm will replace the pad token with the actual image,
111 + # which may be a placeholder image, later.
112 + ]
113 + vllm_outputs = vllm_model.encode(texts, images=input_images)
114 +
115 + hf_outputs = []
116 + with hf_runner(model,
117 + dtype=dtype,
118 + auto_cls=Qwen2VLForConditionalGeneration) as hf_model:
119 + hf_model.postprocess_inputs = partial(
120 + postprocess_inputs,
121 + hf_model,
122 + cache_position=torch.arange(
123 + 0,
124 + 1, # 1 for batch size
125 + requires_grad=False),
126 + use_cache=False)
127 + for text, image, embed_text in zip(input_texts, input_images,
128 + embed_texts):
129 + # dse requires non-standard input processing
130 + # because it needs an image_pad token
131 + messages = get_messages(image, text, embed_text)
132 + prompt = apply_chat_template_and_add_eos(
133 + messages, hf_model.processor.apply_chat_template)
134 + inputs = hf_model.get_inputs(
135 + prompts=[[prompt]],
136 + images=[[image]],
137 + )
138 + with torch.no_grad():
139 + outputs = hf_model.model(
140 + **hf_model.wrap_device(inputs[0],
141 + device=hf_model.model.device.type),
142 + return_dict=True,
143 + output_hidden_states=True,
144 + )
145 + pooled_output = torch.nn.functional.normalize(
146 + outputs.hidden_states[-1][0, -1], p=2, dim=-1)
147 + hf_outputs.append(pooled_output.tolist())
148 +
149 + check_embeddings_close(
150 + embeddings_0_lst=hf_outputs,
151 + embeddings_1_lst=vllm_outputs,
152 + name_0="hf",
153 + name_1="vllm",
154 + )
155 +
156 +
157 +@pytest.mark.parametrize("model", MODELS)
158 +@pytest.mark.parametrize("dtype", ["bfloat16"])
159 +def test_models_text(
160 + hf_runner,
161 + vllm_runner,
162 + image_assets,
163 + model: str,
164 + dtype: str,
165 +) -> None:
166 + input_texts_images = [(text, image_placeholder)
167 + for text, image_placeholder in HF_TEXT_PROMPTS]
168 + input_texts = [text for text, _ in input_texts_images]
169 + input_images = [image for _, image in input_texts_images]
170 + embed_texts = [True] * len(input_texts)
171 +
172 + _run_test(
173 + hf_runner,
174 + vllm_runner,
175 + input_texts,
176 + input_images, # type: ignore
177 + embed_texts,
178 + model,
179 + dtype=dtype,
180 + )
181 +
182 +
183 +@large_gpu_test(min_gb=48)
184 +@pytest.mark.parametrize("model", MODELS)
185 +@pytest.mark.parametrize("dtype", ["bfloat16"])
186 +def test_models_image(
187 + hf_runner,
188 + vllm_runner,
189 + image_assets,
190 + model: str,
191 + dtype: str,
192 +) -> None:
193 + input_texts_images = [
194 + (text, asset.pil_image)
195 + for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
196 + ]
197 + input_texts = [text for text, _ in input_texts_images]
198 + input_images = [image for _, image in input_texts_images]
199 + embed_texts = [False] * len(input_texts)
200 +
201 + _run_test(
202 + hf_runner,
203 + vllm_runner,
204 + input_texts,
205 + input_images,
206 + embed_texts,
207 + model,
208 + dtype=dtype,
209 + )
vllm/model_executor/models/qwen2_vl.py
+16 -1
@@ -51,6 +51,7 @@ from vllm.model_executor.layers.activation import QuickGELU
51 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
52 RowParallelLinear)
53 from vllm.model_executor.layers.logits_processor import LogitsProcessor
54 +from vllm.model_executor.layers.pooler import Pooler, PoolingType
55 from vllm.model_executor.layers.quantization import (GPTQConfig,
56 GPTQMarlinConfig,
57 QuantizationConfig)
@@ -58,12 +59,13 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
59 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
60 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
61 from vllm.model_executor.models.qwen2 import Qwen2Model
62 +from vllm.model_executor.pooling_metadata import PoolingMetadata
63 from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
64 MultiModalKwargs)
65 from vllm.multimodal.base import MultiModalData
66 from vllm.multimodal.image import cached_get_image_processor
67 from vllm.multimodal.utils import cached_get_tokenizer
66 -from vllm.sequence import IntermediateTensors, SequenceData
68 +from vllm.sequence import IntermediateTensors, PoolerOutput, SequenceData
69 from vllm.transformers_utils.config import uses_mrope
70 from vllm.transformers_utils.processor import cached_get_processor
71
@@ -1067,6 +1069,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1069 config = vllm_config.model_config.hf_config
1070 cache_config = vllm_config.cache_config
1071 quant_config = vllm_config.quant_config
1072 + pooler_config = vllm_config.model_config.pooler_config
1073 multimodal_config = vllm_config.model_config.multimodal_config
1074 assert not cache_config.enable_prefix_caching, \
1075 "Qwen2-VL currently does not support prefix caching"
@@ -1098,6 +1101,11 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1101
1102 self.logits_processor = LogitsProcessor(config.vocab_size)
1103 self.sampler = get_sampler()
1104 + self._pooler = Pooler.from_config_with_defaults(
1105 + pooler_config,
1106 + pooling_type=PoolingType.LAST,
1107 + normalize=True,
1108 + softmax=False)
1109 self.make_empty_intermediate_tensors = (
1110 make_empty_intermediate_tensors_factory(
1111 ["hidden_states", "residual"], config.hidden_size))
@@ -1318,6 +1326,13 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1326 next_tokens = self.sampler(logits, sampling_metadata)
1327 return next_tokens
1328
1329 + def pooler(
1330 + self,
1331 + hidden_states: torch.Tensor,
1332 + pooling_metadata: PoolingMetadata,
1333 + ) -> Optional[PoolerOutput]:
1334 + return self._pooler(hidden_states, pooling_metadata)
1335 +
1336 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1337 stacked_params_mapping = [
1338 # (param_name, shard_name, shard_id)
vllm/model_executor/models/registry.py
+1
@@ -109,6 +109,7 @@ _EMBEDDING_MODELS = {
109 # [Multimodal]
110 "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
111 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
112 + "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration") # noqa: E501,
113 }
114
115 _MULTIMODAL_MODELS = {