A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Frontend] Add missing chat templates for various MLLMs (#17758)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
May 7, 2025 at 15:10 UTC
8a15c2603a902820203040a4773ac0e2335e693d
11 files changed
+30
-33
docs/source/serving/multimodal_inputs.md
+1
-1
@@ -216,7 +216,7 @@ A chat template is **required** to use Chat Completions API.
216
217
Although most models come with a chat template, for others you have to define one yourself.
218
The chat template can be inferred based on the documentation on the model's HuggingFace repo.
219
-For example, LLaVA-1.5 (`llava-hf/llava-1.5-7b-hf`) requires a chat template that can be found here: <gh-file:examples/template_llava.jinja>
219
+For example, DeepSeek-VL2 requires a chat template that can be found here: <gh-file:examples/template_deepseek_vl2.jinja>
220
:::
221
222
### Image Inputs
examples/online_serving/openai_chat_completion_client_for_multimodal.py
+1
-1
@@ -5,7 +5,7 @@ and run online serving with OpenAI client.
5
Launch the vLLM server with the following command:
6
7
(single image inference with Llava)
8
-vllm serve llava-hf/llava-1.5-7b-hf --chat-template template_llava.jinja
8
+vllm serve llava-hf/llava-1.5-7b-hf
9
10
(multi-image inference with Phi-3.5-vision-instruct)
11
vllm serve microsoft/Phi-3.5-vision-instruct --task generate \
examples/template_chameleon.jinja
new
+3
@@ -0,0 +1,3 @@
1
+{%- for message in messages -%}
2
+ {{- message['content'] -}}
3
+{%- endfor -%}
examples/template_florence2.jinja
+1
-5
@@ -1,7 +1,3 @@
1
{%- for message in messages -%}
2
- {%- if message['role'] == 'user' -%}
3
- {{- message['content'] -}}
4
- {%- elif message['role'] == 'assistant' -%}
5
- {{- message['content'] -}}
6
- {%- endif -%}
2
+ {{- message['content'] -}}
3
{%- endfor -%}
examples/template_fuyu.jinja
new
+3
@@ -0,0 +1,3 @@
1
+{%- for message in messages -%}
2
+ {{- message['content'] + '\n' -}}
3
+{%- endfor -%}
examples/template_llava.jinja
deleted
-23
@@ -1,23 +0,0 @@
1
-{%- if messages[0]['role'] == 'system' -%}
2
- {%- set system_message = messages[0]['content'] -%}
3
- {%- set messages = messages[1:] -%}
4
-{%- else -%}
5
- {% set system_message = '' -%}
6
-{%- endif -%}
7
-
8
-{{ bos_token + system_message }}
9
-{%- for message in messages -%}
10
- {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
11
- {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
12
- {%- endif -%}
13
-
14
- {%- if message['role'] == 'user' -%}
15
- {{ 'USER: ' + message['content'] + '\n' }}
16
- {%- elif message['role'] == 'assistant' -%}
17
- {{ 'ASSISTANT: ' + message['content'] + eos_token + '\n' }}
18
- {%- endif -%}
19
-{%- endfor -%}
20
-
21
-{%- if add_generation_prompt -%}
22
- {{ 'ASSISTANT:' }}
23
-{% endif %}
examples/template_paligemma.jinja
new
+3
@@ -0,0 +1,3 @@
1
+{%- for message in messages -%}
2
+ {{- message['content'] -}}
3
+{%- endfor -%}
examples/template_qwen_vl.jinja
new
+3
@@ -0,0 +1,3 @@
1
+{%- for message in messages -%}
2
+ {{- message['content'] -}}
3
+{%- endfor -%}
examples/template_qwen_vl_chat.jinja
new
+10
@@ -0,0 +1,10 @@
1
+{%- for message in messages -%}
2
+ {{- '<|im_start|>' + message['role'] + '\n' + message['content'] -}}
3
+ {%- if (loop.last and add_generation_prompt) or not loop.last -%}
4
+ {{- '<|im_end|>' + '\n' -}}
5
+ {%- endif -%}
6
+{%- endfor -%}
7
+
8
+{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%}
9
+ {{- '<|im_start|>assistant\n' -}}
10
+{%- endif -%}
tests/entrypoints/test_chat_utils.py
+5
-1
@@ -900,6 +900,7 @@ def test_resolve_content_format_hf_defined(model, expected_format):
900
[("template_alpaca.jinja", "string"),
901
("template_baichuan.jinja", "string"),
902
("template_blip2.jinja", "string"),
903
+ ("template_chameleon.jinja", "string"),
904
("template_chatglm.jinja", "string"),
905
("template_chatglm2.jinja", "string"),
906
("template_chatml.jinja", "string"),
@@ -908,9 +909,12 @@ def test_resolve_content_format_hf_defined(model, expected_format):
909
("template_falcon_180b.jinja", "string"),
910
("template_falcon.jinja", "string"),
911
("template_florence2.jinja", "string"),
912
+ ("template_fuyu.jinja", "string"),
913
("template_inkbot.jinja", "string"),
912
- ("template_llava.jinja", "string"),
914
+ ("template_paligemma.jinja", "string"),
915
("template_teleflm.jinja", "string"),
916
+ ("template_qwen_vl.jinja", "string"),
917
+ ("template_qwen_vl_chat.jinja", "string"),
918
("template_vlm2vec.jinja", "openai"),
919
("tool_chat_template_granite_20b_fc.jinja", "string"),
920
("tool_chat_template_hermes.jinja", "string"),
tests/v1/tpu/test_multimodal.py
-2
@@ -64,8 +64,6 @@ async def test_basic_vision(model_name: str, base64_encoded_image: dict[str,
64
"576",
65
# NOTE: max-num-batched-tokens>=mm_item_size
66
"--disable_chunked_mm_input",
67
- "--chat-template",
68
- "examples/template_llava.jinja"
67
]
68
69
# Server will pre-compile on first startup (takes a long time).