A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Frontend] Use a proper chat template for VLM2Vec (#9912)
Cyrus Leung committed
Nov 1, 2024 at 22:09 UTC
ba0d8920742597269745f3551eb97b1b19f5e582
6 files changed
+78
-11
docs/source/models/vlm.rst
+10
-4
@@ -240,8 +240,7 @@ To consume the server, you can use the OpenAI client like in the example below:
240
)
241
print("Chat completion output:", chat_response.choices[0].message.content)
242
243
-
244
-A full code example can be found in `examples/openai_api_client_for_multimodal.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_api_client_for_multimodal.py>`_.
243
+A full code example can be found in `examples/openai_chat_completion_client_for_multimodal.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_completion_client_for_multimodal.py>`_.
244
245
.. tip::
246
There is no need to place image placeholders in the text content of the API request - they are already represented by the image content.
@@ -269,14 +268,19 @@ In this example, we will serve the ``TIGER-Lab/VLM2Vec-Full`` model.
268
.. code-block:: bash
269
270
vllm serve TIGER-Lab/VLM2Vec-Full --task embedding \
272
- --trust-remote-code --max-model-len 4096
271
+ --trust-remote-code --max-model-len 4096 --chat-template examples/template_vlm2vec.jinja
272
273
.. important::
274
275
Since VLM2Vec has the same model architecture as Phi-3.5-Vision, we have to explicitly pass ``--task embedding``
276
to run this model in embedding mode instead of text generation mode.
277
279
-Since this schema is not defined by OpenAI client, we post a request to the server using the lower-level ``requests`` library:
278
+.. important::
279
+
280
+ VLM2Vec does not expect chat-based input. We use a `custom chat template <https://github.com/vllm-project/vllm/blob/main/examples/template_vlm2vec.jinja>`_
281
+ to combine the text and images together.
282
+
283
+Since the request schema is not defined by OpenAI client, we post a request to the server using the lower-level ``requests`` library:
284
285
.. code-block:: python
286
@@ -301,3 +305,5 @@ Since this schema is not defined by OpenAI client, we post a request to the serv
305
response.raise_for_status()
306
response_json = response.json()
307
print("Embedding output:", response_json["data"][0]["embedding"])
308
+
309
+A full code example can be found in `examples/openai_chat_embedding_client_for_multimodal.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_chat_embedding_client_for_multimodal.py>`_.
examples/openai_chat_completion_client_for_multimodal.py
renamed
examples/openai_chat_embedding_client_for_multimodal.py
new
+33
@@ -0,0 +1,33 @@
1
+import requests
2
+
3
+image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
4
+
5
+response = requests.post(
6
+ "http://localhost:8000/v1/embeddings",
7
+ json={
8
+ "model":
9
+ "TIGER-Lab/VLM2Vec-Full",
10
+ "messages": [{
11
+ "role":
12
+ "user",
13
+ "content": [
14
+ {
15
+ "type": "image_url",
16
+ "image_url": {
17
+ "url": image_url
18
+ }
19
+ },
20
+ {
21
+ "type": "text",
22
+ "text": "Represent the given image."
23
+ },
24
+ ],
25
+ }],
26
+ "encoding_format":
27
+ "float",
28
+ },
29
+)
30
+response.raise_for_status()
31
+response_json = response.json()
32
+
33
+print("Embedding output:", response_json["data"][0]["embedding"])
examples/template_vlm2vec.jinja
new
+16
@@ -0,0 +1,16 @@
1
+{%- if messages | length > 1 -%}
2
+ {{ raise_exception('Embedding models should only embed one message at a time') }}
3
+{%- endif -%}
4
+
5
+{% set vars = namespace(parts=[], next_image_id=1) %}
6
+{%- for message in messages -%}
7
+ {%- for content in message['content'] -%}
8
+ {%- if content['type'] == 'text' -%}
9
+ {%- set vars.parts = vars.parts + [content['text']] %}
10
+ {%- elif content['type'] == 'image' -%}
11
+ {%- set vars.parts = vars.parts + ['<|image_{i:d}|>'.format(i=vars.next_image_id)] %}
12
+ {%- set vars.next_image_id = vars.next_image_id + 1 %}
13
+ {%- endif -%}
14
+ {%- endfor -%}
15
+{%- endfor -%}
16
+{{ vars.parts | join(' ') }}
tests/entrypoints/openai/test_vision_embedding.py
+8
-3
@@ -6,11 +6,14 @@ import requests
6
7
from vllm.multimodal.utils import encode_image_base64, fetch_image
8
9
-from ...utils import RemoteOpenAIServer
9
+from ...utils import VLLM_PATH, RemoteOpenAIServer
10
11
MODEL_NAME = "TIGER-Lab/VLM2Vec-Full"
12
MAXIMUM_IMAGES = 2
13
14
+vlm2vec_jinja_path = VLLM_PATH / "examples/template_vlm2vec.jinja"
15
+assert vlm2vec_jinja_path.exists()
16
+
17
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
18
TEST_IMAGE_URLS = [
19
"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
@@ -35,6 +38,8 @@ def server():
38
"--trust-remote-code",
39
"--limit-mm-per-prompt",
40
f"image={MAXIMUM_IMAGES}",
41
+ "--chat-template",
42
+ str(vlm2vec_jinja_path),
43
]
44
45
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -90,5 +95,5 @@ async def test_image_embedding(server: RemoteOpenAIServer, model_name: str,
95
assert len(embeddings["data"]) == 1
96
assert len(embeddings["data"][0]["embedding"]) == 3072
97
assert embeddings["usage"]["completion_tokens"] == 0
93
- assert embeddings["usage"]["prompt_tokens"] == 771
94
- assert embeddings["usage"]["total_tokens"] == 771
98
+ assert embeddings["usage"]["prompt_tokens"] == 762
99
+ assert embeddings["usage"]["total_tokens"] == 762
vllm/entrypoints/chat_utils.py
+11
-4
@@ -156,6 +156,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
156
157
self._items: List[_T] = []
158
159
+ @property
160
+ def model_config(self) -> ModelConfig:
161
+ return self._model_config
162
+
163
@staticmethod
164
@lru_cache(maxsize=None)
165
def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str:
@@ -491,10 +495,13 @@ def _parse_chat_message_content_parts(
495
content: List[Union[str, Dict[str, str]]] = []
496
497
mm_parser = mm_tracker.create_parser()
494
- wrap_dicts = \
495
- mm_tracker._model_config.hf_config.model_type in \
496
- MODEL_KEEP_MULTI_MODAL_CONTENT or \
497
- (chat_template_text_format == "openai")
498
+ model_config = mm_tracker.model_config
499
+
500
+ wrap_dicts = (chat_template_text_format == "openai"
501
+ or (model_config.task == "embedding"
502
+ and model_config.is_multimodal_model)
503
+ or (model_config.hf_config.model_type
504
+ in MODEL_KEEP_MULTI_MODAL_CONTENT))
505
506
for part in parts:
507
parse_res = _parse_chat_message_content_part(