A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Feature] Add support for Llama 3.1 and 3.2 tool use (#8343)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Maximilien de Bayser committed
Sep 26, 2024 at 21:01 UTC
344cd2b6f4c22bf278cff96066001d216ec1fe82
10 files changed
+576
-27
docs/source/serving/openai_compatible_server.md
+24
-2
@@ -157,10 +157,10 @@ vLLM will use guided decoding to ensure the response matches the tool parameter
157
To enable this feature, you should set the following flags:
158
* `--enable-auto-tool-choice` -- **mandatory** Auto tool choice. tells vLLM that you want to enable the model to generate its own tool calls when it
159
deems appropriate.
160
-* `--tool-call-parser` -- select the tool parser to use - currently either `hermes` or `mistral`. Additional tool parsers
160
+* `--tool-call-parser` -- select the tool parser to use - currently either `hermes`, `mistral` or `llama3_json`. Additional tool parsers
161
will continue to be added in the future.
162
* `--chat-template` -- **optional** for auto tool choice. the path to the chat template which handles `tool`-role messages and `assistant`-role messages
163
-that contain previously generated tool calls. Hermes and Mistral models have tool-compatible chat templates in their
163
+that contain previously generated tool calls. Hermes, Mistral and Llama models have tool-compatible chat templates in their
164
`tokenizer_config.json` files, but you can specify a custom template. This argument can be set to `tool_use` if your model has a tool use-specific chat
165
template configured in the `tokenizer_config.json`. In this case, it will be used per the `transformers` specification. More on this [here](https://huggingface.co/docs/transformers/en/chat_templating#why-do-some-models-have-multiple-templates)
166
from HuggingFace; and you can find an example of this in a `tokenizer_config.json` [here](https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-8B/blob/main/tokenizer_config.json)
@@ -197,3 +197,25 @@ when tools are provided, that results in much better reliability when working wi
197
198
199
Recommended flags: `--tool-call-parser mistral --chat-template examples/tool_chat_template_mistral_parallel.jinja`
200
+
201
+#### Llama Models
202
+Supported models:
203
+* `meta-llama/Meta-Llama-3.1-8B-Instruct`
204
+* `meta-llama/Meta-Llama-3.1-70B-Instruct`
205
+* `meta-llama/Meta-Llama-3.1-405B-Instruct`
206
+* `meta-llama/Meta-Llama-3.1-405B-Instruct-FP8`
207
+
208
+The tool calling that is supported is the [JSON based tool calling](https://llama.meta.com/docs/model-cards-and-prompt-formats/llama3_1/#json-based-tool-calling).
209
+Other tool calling formats like the built in python tool calling or custom tool calling are not supported.
210
+
211
+Known issues:
212
+1. Parallel tool calls are not supported.
213
+2. The model can generate parameters with a wrong format, such as generating
214
+ an array serialized as string instead of an array.
215
+
216
+The `tool_chat_template_llama3_json.jinja` file contains the "official" Llama chat template, but tweaked so that
217
+it works better with vLLM.
218
+
219
+Recommended flags: `--tool-call-parser llama3_json --chat-template examples/tool_chat_template_llama3_json.jinja`
220
+
221
+
examples/tool_chat_template_llama3.1_json.jinja
new
+94
@@ -0,0 +1,94 @@
1
+{{- bos_token }}
2
+{%- if custom_tools is defined %}
3
+ {%- set tools = custom_tools %}
4
+{%- endif %}
5
+{%- if not tools_in_user_message is defined %}
6
+ {#- Llama 3.1 doesn't pass all tests if the tools are in the system prompt #}
7
+ {%- set tools_in_user_message = true %}
8
+{%- endif %}
9
+{%- if not date_string is defined %}
10
+ {%- if strftime_now is defined %}
11
+ {%- set date_string = strftime_now("%d %b %Y") %}
12
+ {%- else %}
13
+ {%- set date_string = "26 Jul 2024" %}
14
+ {%- endif %}
15
+{%- endif %}
16
+{%- if not tools is defined %}
17
+ {%- set tools = none %}
18
+{%- endif %}
19
+
20
+{#- This block extracts the system message, so we can slot it into the right place. #}
21
+{%- if messages[0]['role'] == 'system' %}
22
+ {%- set system_message = messages[0]['content']|trim %}
23
+ {%- set messages = messages[1:] %}
24
+{%- else %}
25
+ {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
26
+{%- endif %}
27
+
28
+{#- System message #}
29
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
30
+{%- if tools is not none %}
31
+ {{- "Environment: ipython\n" }}
32
+{%- endif %}
33
+{{- "Cutting Knowledge Date: December 2023\n" }}
34
+{{- "Today Date: " + date_string + "\n\n" }}
35
+{%- if tools is not none and not tools_in_user_message %}
36
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
37
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
38
+ {{- "Do not use variables.\n\n" }}
39
+ {%- for t in tools %}
40
+ {{- t | tojson(indent=4) }}
41
+ {{- "\n\n" }}
42
+ {%- endfor %}
43
+{%- endif %}
44
+{{- system_message }}
45
+{{- "<|eot_id|>" }}
46
+
47
+{#- Custom tools are passed in a user message with some extra guidance #}
48
+{%- if tools_in_user_message and not tools is none %}
49
+ {#- Extract the first user message so we can plug it in here #}
50
+ {%- if messages | length != 0 %}
51
+ {%- set first_user_message = messages[0]['content']|trim %}
52
+ {%- set messages = messages[1:] %}
53
+ {%- else %}
54
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
55
+ {%- endif %}
56
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
57
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
58
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
59
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
60
+ {{- "Do not use variables.\n\n" }}
61
+ {%- for t in tools %}
62
+ {{- t | tojson(indent=4) }}
63
+ {{- "\n\n" }}
64
+ {%- endfor %}
65
+ {{- first_user_message + "<|eot_id|>"}}
66
+{%- endif %}
67
+
68
+{%- for message in messages %}
69
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
70
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
71
+ {%- elif 'tool_calls' in message %}
72
+ {%- if not message.tool_calls|length == 1 %}
73
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
74
+ {%- endif %}
75
+ {%- set tool_call = message.tool_calls[0].function %}
76
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
77
+ {{- '{"name": "' + tool_call.name + '", ' }}
78
+ {{- '"parameters": ' }}
79
+ {{- tool_call.arguments | tojson }}
80
+ {{- "}" }}
81
+ {{- "<|eot_id|>" }}
82
+ {%- elif message.role == "tool" or message.role == "ipython" %}
83
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
84
+ {%- if message.content is mapping %}
85
+ {{- message.content | tojson }}
86
+ {%- else %}
87
+ {{- { "output": message.content } | tojson }}
88
+ {%- endif %}
89
+ {{- "<|eot_id|>" }}
90
+ {%- endif %}
91
+{%- endfor %}
92
+{%- if add_generation_prompt %}
93
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
94
+{%- endif %}
examples/tool_chat_template_llama3.2_json.jinja
new
+93
@@ -0,0 +1,93 @@
1
+{{- bos_token }}
2
+{%- if custom_tools is defined %}
3
+ {%- set tools = custom_tools %}
4
+{%- endif %}
5
+{%- if not tools_in_user_message is defined %}
6
+ {%- set tools_in_user_message = false %}
7
+{%- endif %}
8
+{%- if not date_string is defined %}
9
+ {%- if strftime_now is defined %}
10
+ {%- set date_string = strftime_now("%d %b %Y") %}
11
+ {%- else %}
12
+ {%- set date_string = "26 Jul 2024" %}
13
+ {%- endif %}
14
+{%- endif %}
15
+{%- if not tools is defined %}
16
+ {%- set tools = none %}
17
+{%- endif %}
18
+
19
+{#- This block extracts the system message, so we can slot it into the right place. #}
20
+{%- if messages[0]['role'] == 'system' %}
21
+ {%- set system_message = messages[0]['content']|trim %}
22
+ {%- set messages = messages[1:] %}
23
+{%- else %}
24
+ {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
25
+{%- endif %}
26
+
27
+{#- System message #}
28
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
29
+{%- if tools is not none %}
30
+ {{- "Environment: ipython\n" }}
31
+{%- endif %}
32
+{{- "Cutting Knowledge Date: December 2023\n" }}
33
+{{- "Today Date: " + date_string + "\n\n" }}
34
+{%- if tools is not none and not tools_in_user_message %}
35
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
36
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
37
+ {{- "Do not use variables.\n\n" }}
38
+ {%- for t in tools %}
39
+ {{- t | tojson(indent=4) }}
40
+ {{- "\n\n" }}
41
+ {%- endfor %}
42
+{%- endif %}
43
+{{- system_message }}
44
+{{- "<|eot_id|>" }}
45
+
46
+{#- Custom tools are passed in a user message with some extra guidance #}
47
+{%- if tools_in_user_message and not tools is none %}
48
+ {#- Extract the first user message so we can plug it in here #}
49
+ {%- if messages | length != 0 %}
50
+ {%- set first_user_message = messages[0]['content']|trim %}
51
+ {%- set messages = messages[1:] %}
52
+ {%- else %}
53
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
54
+ {%- endif %}
55
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
56
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
57
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
58
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
59
+ {{- "Do not use variables.\n\n" }}
60
+ {%- for t in tools %}
61
+ {{- t | tojson(indent=4) }}
62
+ {{- "\n\n" }}
63
+ {%- endfor %}
64
+ {{- first_user_message + "<|eot_id|>"}}
65
+{%- endif %}
66
+
67
+{%- for message in messages %}
68
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
69
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
70
+ {%- elif 'tool_calls' in message %}
71
+ {%- if not message.tool_calls|length == 1 %}
72
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
73
+ {%- endif %}
74
+ {%- set tool_call = message.tool_calls[0].function %}
75
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
76
+ {{- '{"name": "' + tool_call.name + '", ' }}
77
+ {{- '"parameters": ' }}
78
+ {{- tool_call.arguments | tojson }}
79
+ {{- "}" }}
80
+ {{- "<|eot_id|>" }}
81
+ {%- elif message.role == "tool" or message.role == "ipython" %}
82
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
83
+ {%- if message.content is mapping %}
84
+ {{- message.content | tojson }}
85
+ {%- else %}
86
+ {{- { "output": message.content } | tojson }}
87
+ {%- endif %}
88
+ {{- "<|eot_id|>" }}
89
+ {%- endif %}
90
+{%- endfor %}
91
+{%- if add_generation_prompt %}
92
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
93
+{%- endif %}
tests/tool_use/test_chat_completions.py
+10
-7
@@ -3,18 +3,20 @@ from typing import List
3
import openai
4
import pytest
5
6
-from .utils import MESSAGES_WITHOUT_TOOLS, WEATHER_TOOL
6
+from .utils import (MESSAGES_WITHOUT_TOOLS, WEATHER_TOOL, ServerConfig,
7
+ ensure_system_prompt)
8
9
10
# test: make sure chat completions without tools provided work even when tools
11
# are enabled. This makes sure tool call chat templates work, AND that the tool
12
# parser stream processing doesn't change the output of the model.
13
@pytest.mark.asyncio
13
-async def test_chat_completion_without_tools(client: openai.AsyncOpenAI):
14
+async def test_chat_completion_without_tools(client: openai.AsyncOpenAI,
15
+ server_config: ServerConfig):
16
models = await client.models.list()
17
model_name: str = models.data[0].id
18
chat_completion = await client.chat.completions.create(
17
- messages=MESSAGES_WITHOUT_TOOLS,
19
+ messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
20
temperature=0,
21
max_tokens=150,
22
model=model_name,
@@ -34,7 +36,7 @@ async def test_chat_completion_without_tools(client: openai.AsyncOpenAI):
36
37
# make the same request, streaming
38
stream = await client.chat.completions.create(
37
- messages=MESSAGES_WITHOUT_TOOLS,
39
+ messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
40
temperature=0,
41
max_tokens=150,
42
model=model_name,
@@ -77,11 +79,12 @@ async def test_chat_completion_without_tools(client: openai.AsyncOpenAI):
79
# tools, to make sure we can still get normal chat completion responses
80
# and that they won't be parsed as tools
81
@pytest.mark.asyncio
80
-async def test_chat_completion_with_tools(client: openai.AsyncOpenAI):
82
+async def test_chat_completion_with_tools(client: openai.AsyncOpenAI,
83
+ server_config: ServerConfig):
84
models = await client.models.list()
85
model_name: str = models.data[0].id
86
chat_completion = await client.chat.completions.create(
84
- messages=MESSAGES_WITHOUT_TOOLS,
87
+ messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
88
temperature=0,
89
max_tokens=150,
90
model=model_name,
@@ -102,7 +105,7 @@ async def test_chat_completion_with_tools(client: openai.AsyncOpenAI):
105
106
# make the same request, streaming
107
stream = await client.chat.completions.create(
105
- messages=MESSAGES_WITHOUT_TOOLS,
108
+ messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
109
temperature=0,
110
max_tokens=150,
111
model=model_name,
tests/tool_use/test_parallel_tool_calls.py
+15
-3
@@ -6,7 +6,7 @@ import pytest
6
7
from .utils import (MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
8
MESSAGES_WITH_PARALLEL_TOOL_RESPONSE, SEARCH_TOOL,
9
- WEATHER_TOOL)
9
+ WEATHER_TOOL, ServerConfig)
10
11
12
# test: getting the model to generate parallel tool calls (streaming/not)
@@ -14,7 +14,13 @@ from .utils import (MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
14
# may be added in the future. e.g. llama 3.1 models are not designed to support
15
# parallel tool calls.
16
@pytest.mark.asyncio
17
-async def test_parallel_tool_calls(client: openai.AsyncOpenAI):
17
+async def test_parallel_tool_calls(client: openai.AsyncOpenAI,
18
+ server_config: ServerConfig):
19
+
20
+ if not server_config.get("supports_parallel", True):
21
+ pytest.skip("The {} model doesn't support parallel tool calls".format(
22
+ server_config["model"]))
23
+
24
models = await client.models.list()
25
model_name: str = models.data[0].id
26
chat_completion = await client.chat.completions.create(
@@ -136,7 +142,13 @@ async def test_parallel_tool_calls(client: openai.AsyncOpenAI):
142
# test: providing parallel tool calls back to the model to get a response
143
# (streaming/not)
144
@pytest.mark.asyncio
139
-async def test_parallel_tool_calls_with_results(client: openai.AsyncOpenAI):
145
+async def test_parallel_tool_calls_with_results(client: openai.AsyncOpenAI,
146
+ server_config: ServerConfig):
147
+
148
+ if not server_config.get("supports_parallel", True):
149
+ pytest.skip("The {} model doesn't support parallel tool calls".format(
150
+ server_config["model"]))
151
+
152
models = await client.models.list()
153
model_name: str = models.data[0].id
154
chat_completion = await client.chat.completions.create(
tests/tool_use/utils.py
+58
-13
@@ -1,4 +1,5 @@
1
-from typing import Dict, List
1
+from copy import deepcopy
2
+from typing import Any, Dict, List, Optional
3
4
from openai.types.chat import (ChatCompletionMessageParam,
5
ChatCompletionToolParam)
@@ -7,9 +8,30 @@ from typing_extensions import TypedDict
8
from tests.utils import VLLM_PATH
9
10
10
-class ServerConfig(TypedDict):
11
+class ServerConfig(TypedDict, total=False):
12
model: str
13
arguments: List[str]
14
+ system_prompt: Optional[str]
15
+ supports_parallel: Optional[bool]
16
+
17
+
18
+def patch_system_prompt(messages: List[Dict[str, Any]],
19
+ system_prompt: str) -> List[Dict[str, Any]]:
20
+ new_messages = deepcopy(messages)
21
+ if new_messages[0]["role"] == "system":
22
+ new_messages[0]["content"] = system_prompt
23
+ else:
24
+ new_messages.insert(0, {"role": "system", "content": system_prompt})
25
+ return new_messages
26
+
27
+
28
+def ensure_system_prompt(messages: List[Dict[str, Any]],
29
+ config: ServerConfig) -> List[Dict[str, Any]]:
30
+ prompt = config.get("system_prompt")
31
+ if prompt:
32
+ return patch_system_prompt(messages, prompt)
33
+ else:
34
+ return messages
35
36
37
# universal args for all models go here. also good if you need to test locally
@@ -23,7 +45,33 @@ CONFIGS: Dict[str, ServerConfig] = {
45
"arguments": [
46
"--tool-call-parser", "hermes", "--chat-template",
47
str(VLLM_PATH / "examples/tool_chat_template_hermes.jinja")
26
- ]
48
+ ],
49
+ "system_prompt":
50
+ "You are a helpful assistant with access to tools. If a tool"
51
+ " that you have would be helpful to answer a user query, "
52
+ "call the tool. Otherwise, answer the user's query directly "
53
+ "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "
54
+ "to the user's question - just respond to it normally."
55
+ },
56
+ "llama": {
57
+ "model":
58
+ "meta-llama/Meta-Llama-3.1-8B-Instruct",
59
+ "arguments": [
60
+ "--tool-call-parser", "llama3_json", "--chat-template",
61
+ str(VLLM_PATH / "examples/tool_chat_template_llama3.1_json.jinja")
62
+ ],
63
+ "supports_parallel":
64
+ False,
65
+ },
66
+ "llama3.2": {
67
+ "model":
68
+ "meta-llama/Llama-3.2-3B-Instruct",
69
+ "arguments": [
70
+ "--tool-call-parser", "llama3_json", "--chat-template",
71
+ str(VLLM_PATH / "examples/tool_chat_template_llama3.2_json.jinja")
72
+ ],
73
+ "supports_parallel":
74
+ False,
75
},
76
"mistral": {
77
"model":
@@ -32,7 +80,13 @@ CONFIGS: Dict[str, ServerConfig] = {
80
"--tool-call-parser", "mistral", "--chat-template",
81
str(VLLM_PATH / "examples/tool_chat_template_mistral.jinja"),
82
"--ignore-patterns=\"consolidated.safetensors\""
35
- ]
83
+ ],
84
+ "system_prompt":
85
+ "You are a helpful assistant with access to tools. If a tool"
86
+ " that you have would be helpful to answer a user query, "
87
+ "call the tool. Otherwise, answer the user's query directly "
88
+ "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "
89
+ "to the user's question - just respond to it normally."
90
}
91
}
92
@@ -97,15 +151,6 @@ SEARCH_TOOL: ChatCompletionToolParam = {
151
}
152
153
MESSAGES_WITHOUT_TOOLS: List[ChatCompletionMessageParam] = [{
100
- "role":
101
- "system",
102
- "content":
103
- "You are a helpful assistant with access to tools. If a tool"
104
- " that you have would be helpful to answer a user query, "
105
- "call the tool. Otherwise, answer the user's query directly "
106
- "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "
107
- "to the user's question - just respond to it normally."
108
-}, {
154
"role":
155
"user",
156
"content":
vllm/entrypoints/openai/cli_args.py
+1
-1
@@ -193,7 +193,7 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
193
parser.add_argument(
194
"--tool-call-parser",
195
type=str,
196
- choices=["mistral", "hermes"],
196
+ choices=["mistral", "hermes", "llama3_json"],
197
default=None,
198
help=
199
"Select the tool call parser depending on the model that you're using."
vllm/entrypoints/openai/serving_chat.py
+3
@@ -30,6 +30,7 @@ from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
30
PromptAdapterPath,
31
TextTokensPrompt)
32
from vllm.entrypoints.openai.tool_parsers import (Hermes2ProToolParser,
33
+ Llama3JsonToolParser,
34
MistralToolParser,
35
ToolParser)
36
from vllm.inputs import TokensPrompt
@@ -85,6 +86,8 @@ class OpenAIServingChat(OpenAIServing):
86
self.tool_parser = MistralToolParser
87
elif tool_parser == "hermes":
88
self.tool_parser = Hermes2ProToolParser
89
+ elif tool_parser == "llama3_json":
90
+ self.tool_parser = Llama3JsonToolParser
91
else:
92
raise TypeError("Error: --enable-auto-tool-choice requires "
93
"--tool-call-parser")
vllm/entrypoints/openai/tool_parsers/__init__.py
+5
-1
@@ -1,5 +1,9 @@
1
from .abstract_tool_parser import ToolParser
2
from .hermes_tool_parser import Hermes2ProToolParser
3
+from .llama_tool_parser import Llama3JsonToolParser
4
from .mistral_tool_parser import MistralToolParser
5
5
-__all__ = ["ToolParser", "Hermes2ProToolParser", "MistralToolParser"]
\ No newline at end of file
6
+__all__ = [
7
+ "ToolParser", "Hermes2ProToolParser", "MistralToolParser",
8
+ "Llama3JsonToolParser"
9
+]
vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py
new
+273
@@ -0,0 +1,273 @@
1
+import json
2
+import re
3
+from json import JSONDecodeError, JSONDecoder
4
+from typing import Dict, List, Sequence, Union
5
+
6
+import partial_json_parser
7
+from partial_json_parser.core.options import Allow
8
+from transformers import PreTrainedTokenizerBase
9
+
10
+from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
11
+ DeltaToolCall,
12
+ ExtractedToolCallInformation,
13
+ FunctionCall, ToolCall)
14
+from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
15
+ ToolParser)
16
+from vllm.entrypoints.openai.tool_parsers.utils import find_common_prefix
17
+from vllm.logger import init_logger
18
+from vllm.utils import random_uuid
19
+
20
+logger = init_logger(__name__)
21
+
22
+
23
+# partial_json_parser doesn't support extra data and
24
+# JSONDecorder.raw_decode doesn't support partial JSON
25
+def partial_json_loads(input_str, flags):
26
+ try:
27
+ return (partial_json_parser.loads(input_str, flags), len(input_str))
28
+ except JSONDecodeError as e:
29
+ if "Extra data" in e.msg:
30
+ dec = JSONDecoder()
31
+ return dec.raw_decode(input_str)
32
+ else:
33
+ raise
34
+
35
+
36
+def is_complete_json(input_str):
37
+ try:
38
+ json.loads(input_str)
39
+ return True
40
+ except JSONDecodeError:
41
+ return False
42
+
43
+
44
+class Llama3JsonToolParser(ToolParser):
45
+ """
46
+ Tool call parser for Llama 3.1 models intended for use with the
47
+ examples/tool_chat_template_llama.jinja template.
48
+
49
+ Used when --enable-auto-tool-choice --tool-call-parser mistral are all set
50
+ """
51
+
52
+ def __init__(self, tokenizer: PreTrainedTokenizerBase):
53
+ super().__init__(tokenizer)
54
+
55
+ # initialize properties used for state when parsing tool calls in
56
+ # streaming mode
57
+ self.prev_tool_call_arr: List[Dict] = []
58
+ self.current_tool_id: int = -1
59
+ self.current_tool_name_sent: bool = False
60
+ self.streamed_args_for_tool: List[str] = [
61
+ ] # map what has been streamed for each tool so far to a list
62
+ self.bot_token = "<|python_tag|>"
63
+ self.bot_token_id = tokenizer.encode(self.bot_token,
64
+ add_special_tokens=False)[0]
65
+ self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL)
66
+
67
+ def extract_tool_calls(self,
68
+ model_output: str) -> ExtractedToolCallInformation:
69
+ """
70
+ Extract the tool calls from a complete model response.
71
+ """
72
+ # case -- if a tool call token is not present, return a text response
73
+ if not (model_output.startswith(self.bot_token)
74
+ or model_output.startswith('{')):
75
+ return ExtractedToolCallInformation(tools_called=False,
76
+ tool_calls=[],
77
+ content=model_output)
78
+
79
+ try:
80
+ # load the JSON, and then use it to build the Function and
81
+ # Tool Call
82
+ dec = JSONDecoder()
83
+ function_call_arr = []
84
+
85
+ # depending on the prompt format the Llama model may or may not
86
+ # prefix the output with the <|python_tag|> token
87
+ start_idx = len(self.bot_token) if model_output.startswith(
88
+ self.bot_token) else 0
89
+ while start_idx < len(model_output):
90
+ (obj, end_idx) = dec.raw_decode(model_output[start_idx:])
91
+ start_idx += end_idx + len('; ')
92
+ function_call_arr.append(obj)
93
+
94
+ tool_calls: List[ToolCall] = [
95
+ ToolCall(
96
+ type="function",
97
+ function=FunctionCall(
98
+ name=raw_function_call["name"],
99
+ # function call args are JSON but as a string
100
+ arguments=json.dumps(raw_function_call["arguments"] \
101
+ if "arguments" in raw_function_call \
102
+ else raw_function_call["parameters"])))
103
+ for raw_function_call in function_call_arr
104
+ ]
105
+
106
+ # get any content before the tool call
107
+ ret = ExtractedToolCallInformation(tools_called=True,
108
+ tool_calls=tool_calls,
109
+ content=None)
110
+ return ret
111
+
112
+ except Exception as e:
113
+ logger.error("Error in extracting tool call from response: %s", e)
114
+ print("ERROR", e)
115
+ # return information to just treat the tool call as regular JSON
116
+ return ExtractedToolCallInformation(tools_called=False,
117
+ tool_calls=[],
118
+ content=model_output)
119
+
120
+ def extract_tool_calls_streaming(
121
+ self,
122
+ previous_text: str,
123
+ current_text: str,
124
+ delta_text: str,
125
+ previous_token_ids: Sequence[int],
126
+ current_token_ids: Sequence[int],
127
+ delta_token_ids: Sequence[int],
128
+ ) -> Union[DeltaMessage, None]:
129
+
130
+ if not (current_text.startswith(self.bot_token)
131
+ or current_text.startswith('{')):
132
+ return DeltaMessage(content=delta_text)
133
+
134
+ # bit mask flags for partial JSON parsing. If the name hasn't been
135
+ # sent yet, don't allow sending
136
+ # an incomplete string since OpenAI only ever (as far as I have
137
+ # seen) allows sending the entire tool/ function name at once.
138
+ flags = Allow.ALL if self.current_tool_name_sent \
139
+ else Allow.ALL & ~Allow.STR
140
+ try:
141
+ tool_call_arr = []
142
+ is_complete = []
143
+ try:
144
+ # depending on the prompt format the Llama model may or may not
145
+ # prefix the output with the <|python_tag|> token
146
+ start_idx = len(self.bot_token) if current_text.startswith(
147
+ self.bot_token) else 0
148
+ while start_idx < len(current_text):
149
+ (obj,
150
+ end_idx) = partial_json_loads(current_text[start_idx:],
151
+ flags)
152
+ is_complete.append(
153
+ is_complete_json(current_text[start_idx:start_idx +
154
+ end_idx]))
155
+ start_idx += end_idx + len('; ')
156
+ # depending on the prompt Llama can use
157
+ # either arguments or parameters
158
+ if "parameters" in obj:
159
+ assert "arguments" not in obj, \
160
+ "model generated both parameters and arguments"
161
+ obj["arguments"] = obj["parameters"]
162
+ tool_call_arr.append(obj)
163
+ except partial_json_parser.core.exceptions.MalformedJSON:
164
+ logger.debug('not enough tokens to parse into JSON yet')
165
+ return None
166
+
167
+ # select as the current tool call the one we're on the state at
168
+ current_tool_call: Dict = tool_call_arr[self.current_tool_id] \
169
+ if len(tool_call_arr) > 0 else {}
170
+
171
+ # case -- if no tokens have been streamed for the tool, e.g.
172
+ # only the array brackets, stream nothing
173
+ if len(tool_call_arr) == 0:
174
+ return None
175
+
176
+ # case: we are starting a new tool in the array
177
+ # -> array has > 0 length AND length has moved past cursor
178
+ elif (len(tool_call_arr) > 0
179
+ and len(tool_call_arr) > self.current_tool_id + 1):
180
+
181
+ # if we're moving on to a new call, first make sure we
182
+ # haven't missed anything in the previous one that was
183
+ # auto-generated due to JSON completions, but wasn't
184
+ # streamed to the client yet.
185
+ if self.current_tool_id >= 0:
186
+ cur_arguments = current_tool_call.get("arguments")
187
+ if cur_arguments:
188
+ cur_args_json = json.dumps(cur_arguments)
189
+ sent = len(
190
+ self.streamed_args_for_tool[self.current_tool_id])
191
+ argument_diff = cur_args_json[sent:]
192
+
193
+ logger.debug("got arguments diff: %s", argument_diff)
194
+ delta = DeltaMessage(tool_calls=[
195
+ DeltaToolCall(index=self.current_tool_id,
196
+ function=DeltaFunctionCall(
197
+ arguments=argument_diff).
198
+ model_dump(exclude_none=True))
199
+ ])
200
+ self.streamed_args_for_tool[
201
+ self.current_tool_id] += argument_diff
202
+ else:
203
+ delta = None
204
+ else:
205
+ delta = None
206
+ # re-set stuff pertaining to progress in the current tool
207
+ self.current_tool_id = len(tool_call_arr) - 1
208
+ self.current_tool_name_sent = False
209
+ self.streamed_args_for_tool.append("")
210
+ logger.debug("starting on new tool %d", self.current_tool_id)
211
+ return delta
212
+
213
+ # if the current tool name hasn't been sent, send if available
214
+ # - otherwise send nothing
215
+ elif not self.current_tool_name_sent:
216
+ function_name = current_tool_call.get("name")
217
+ if function_name:
218
+
219
+ delta = DeltaMessage(tool_calls=[
220
+ DeltaToolCall(index=self.current_tool_id,
221
+ type="function",
222
+ id=f"chatcmpl-tool-{random_uuid()}",
223
+ function=DeltaFunctionCall(
224
+ name=function_name).model_dump(
225
+ exclude_none=True))
226
+ ])
227
+ self.current_tool_name_sent = True
228
+ else:
229
+ delta = None
230
+
231
+ # now we know we're on the same tool call and we're streaming
232
+ # arguments
233
+ else:
234
+ cur_arguments = current_tool_call.get("arguments")
235
+ delta = None
236
+
237
+ if cur_arguments:
238
+ sent = len(
239
+ self.streamed_args_for_tool[self.current_tool_id])
240
+ cur_args_json = json.dumps(cur_arguments)
241
+ prev_arguments = self.prev_tool_call_arr[
242
+ self.current_tool_id].get("arguments")
243
+
244
+ argument_diff = None
245
+ if is_complete[self.current_tool_id]:
246
+ argument_diff = cur_args_json[sent:]
247
+ elif prev_arguments:
248
+ prev_args_json = json.dumps(prev_arguments)
249
+ if cur_args_json != prev_args_json:
250
+
251
+ prefix = find_common_prefix(
252
+ prev_args_json, cur_args_json)
253
+ argument_diff = prefix[sent:]
254
+
255
+ if argument_diff is not None:
256
+ delta = DeltaMessage(tool_calls=[
257
+ DeltaToolCall(index=self.current_tool_id,
258
+ function=DeltaFunctionCall(
259
+ arguments=argument_diff).
260
+ model_dump(exclude_none=True))
261
+ ])
262
+ self.streamed_args_for_tool[
263
+ self.current_tool_id] += argument_diff
264
+
265
+ self.prev_tool_call_arr = tool_call_arr
266
+ return delta
267
+
268
+ except Exception as e:
269
+ logger.error("Error trying to handle streaming tool call: %s", e)
270
+ logger.debug(
271
+ "Skipping chunk as a result of tool streaming extraction "
272
+ "error")
273
+ return None