@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d75ab55f10
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Misc] Add deepseek_vl2 chat template (#12143)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Jan 17, 2025 at 14:34 UTC d75ab55f1035309c96814af46da1c5166209854b
2 files changed +24
examples/template_deepseek_vl2.jinja new
+23
@@ -0,0 +1,23 @@
1 +{%- if messages[0]['role'] == 'system' -%}
2 + {%- set system_message = messages[0]['content'] -%}
3 + {%- set messages = messages[1:] -%}
4 +{%- else -%}
5 + {% set system_message = '' -%}
6 +{%- endif -%}
7 +
8 +{{ bos_token + system_message }}
9 +{%- for message in messages -%}
10 + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
11 + {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
12 + {%- endif -%}
13 +
14 + {%- if message['role'] == 'user' -%}
15 + {{ '<|User|>: ' + message['content'] + '\n' }}
16 + {%- elif message['role'] == 'assistant' -%}
17 + {{ '<|Assistant|>: ' + message['content'] + eos_token + '\n' }}
18 + {%- endif -%}
19 +{%- endfor -%}
20 +
21 +{%- if add_generation_prompt -%}
22 + {{ '<|Assistant|>: ' }}
23 +{% endif %}
tests/entrypoints/test_chat_utils.py
+1
@@ -754,6 +754,7 @@ def test_resolve_content_format_hf_defined(model, expected_format):
754 ("template_chatglm.jinja", "string"),
755 ("template_chatglm2.jinja", "string"),
756 ("template_chatml.jinja", "string"),
757 + ("template_deepseek_vl2.jinja", "string"),
758 ("template_falcon_180b.jinja", "string"),
759 ("template_falcon.jinja", "string"),
760 ("template_inkbot.jinja", "string"),