A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Misc] Update llama 3.2 template to support system prompt with images (#10901)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Dec 4, 2024 at 22:54 UTC
39c89e71a84779c0758ec603efcded7a48bb5fc0
1 file changed
+2
-10
examples/tool_chat_template_llama3.2_json.jinja
+2
-10
@@ -26,13 +26,11 @@
26
{%- endfor %}
27
{%- endfor %}
28
29
-
29
{#- This block extracts the system message, so we can slot it into the right place. #}
30
{%- if messages[0]['role'] == 'system' %}
31
{%- if messages[0]['content'] is string %}
32
{%- set system_message = messages[0]['content']|trim %}
33
{%- else %}
35
- {#- Support vLLM's transforming of a content string to JSON. #}
34
{%- set system_message = messages[0]['content'][0]['text']|trim %}
35
{%- endif %}
36
{%- set messages = messages[1:] %}
@@ -44,14 +42,8 @@
42
{%- endif %}
43
{%- endif %}
44
47
-{#- Including an image is not compatible with a system message #}
48
-{%- if image_ns.has_images and not system_message == "" %}
49
- {{- raise_exception("Prompting with images is incompatible with system messages and tool use.") }}
50
-{%- endif %}
51
-
52
-
53
-{#- System message, if there are no images #}
54
-{%- if not image_ns.has_images %}
45
+{#- System message if there are no images, if the user supplied one, or if tools are used (default tool system message) #}
46
+{%- if system_message or not image_ns.has_images %}
47
{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
48
{%- if tools is not none %}
49
{{- "Environment: ipython\n" }}