A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Frontend][Feature] support tool calling for internlm/internlm2_5-7b-chat model (#8405)
代君 committed
Oct 4, 2024 at 10:36 UTC
3dbb215b38c010c050f7fde3528fe2c6673f7a07
13 files changed
+533
-46
docs/requirements-docs.txt
+2
-1
@@ -12,4 +12,5 @@ torch
12
py-cpuinfo
13
transformers
14
mistral_common >= 1.3.4
15
-openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
\ No newline at end of file
15
+openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
16
+partial-json-parser # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
\ No newline at end of file
docs/source/serving/openai_compatible_server.md
+72
-2
@@ -157,8 +157,9 @@ vLLM will use guided decoding to ensure the response matches the tool parameter
157
To enable this feature, you should set the following flags:
158
* `--enable-auto-tool-choice` -- **mandatory** Auto tool choice. tells vLLM that you want to enable the model to generate its own tool calls when it
159
deems appropriate.
160
-* `--tool-call-parser` -- select the tool parser to use - currently either `hermes`, `mistral` or `llama3_json`. Additional tool parsers
161
-will continue to be added in the future.
160
+* `--tool-call-parser` -- select the tool parser to use - currently either `hermes` or `mistral` or `llama3_json` or `internlm`. Additional tool parsers
161
+will continue to be added in the future, and also can register your own tool parsers in the `--tool-parser-plugin`.
162
+* `--tool-parser-plugin` -- **optional** tool parser plugin used to register user defined tool parsers into vllm, the registered tool parser name can be specified in `--tool-call-parser`.
163
* `--chat-template` -- **optional** for auto tool choice. the path to the chat template which handles `tool`-role messages and `assistant`-role messages
164
that contain previously generated tool calls. Hermes, Mistral and Llama models have tool-compatible chat templates in their
165
`tokenizer_config.json` files, but you can specify a custom template. This argument can be set to `tool_use` if your model has a tool use-specific chat
@@ -218,4 +219,73 @@ it works better with vLLM.
219
220
Recommended flags: `--tool-call-parser llama3_json --chat-template examples/tool_chat_template_llama3_json.jinja`
221
222
+#### Internlm Models
223
+Supported models:
224
+* `internlm/internlm2_5-7b-chat` (confirmed)
225
+* Additional internlm2.5 function-calling models are compatible as well
226
+
227
+Known issues:
228
+* Although this implementation also supports Internlm2, the tool call results are not stable when testing with the `internlm/internlm2-chat-7b` model.
229
+
230
+Recommended flags: `--tool-call-parser internlm --chat-template examples/tool_chat_template_internlm2_tool.jinja`
231
+
232
+
233
+### How to write a tool parser plugin
234
+
235
+A tool parser plugin is a Python file containing one or more ToolParser implementations. You can write a ToolParser similar to the `Hermes2ProToolParser` in vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py.
236
+
237
+Here is a summary of a plugin file:
238
+
239
+```python
240
+
241
+# import the required packages
242
+
243
+# define a tool parser and register it to vllm
244
+# the name list in register_module can be used
245
+# in --tool-call-parser. you can define as many
246
+# tool parsers as you want here.
247
+@ToolParserManager.register_module(["example"])
248
+class ExampleToolParser(ToolParser):
249
+ def __init__(self, tokenizer: AnyTokenizer):
250
+ super().__init__(tokenizer)
251
+
252
+ # adjust request. e.g.: set skip special tokens
253
+ # to False for tool call output.
254
+ def adjust_request(
255
+ self, request: ChatCompletionRequest) -> ChatCompletionRequest:
256
+ return request
257
+
258
+ # implement the tool call parse for stream call
259
+ def extract_tool_calls_streaming(
260
+ self,
261
+ previous_text: str,
262
+ current_text: str,
263
+ delta_text: str,
264
+ previous_token_ids: Sequence[int],
265
+ current_token_ids: Sequence[int],
266
+ delta_token_ids: Sequence[int],
267
+ request: ChatCompletionRequest,
268
+ ) -> Union[DeltaMessage, None]:
269
+ return delta
270
+
271
+ # implement the tool parse for non-stream call
272
+ def extract_tool_calls(
273
+ self,
274
+ model_output: str,
275
+ request: ChatCompletionRequest,
276
+ ) -> ExtractedToolCallInformation:
277
+ return ExtractedToolCallInformation(tools_called=False,
278
+ tool_calls=[],
279
+ content=text)
280
+
281
+
282
+```
283
+
284
+Then you can use this plugin in the command line like this.
285
+```
286
+ --enable-auto-tool-choice \
287
+ --tool-parser-plugin <absolute path of the plugin file>
288
+ --tool-call-parser example \
289
+ --chat-template <your chat template> \
290
+```
291
examples/tool_chat_template_internlm2_tool.jinja
new
+60
@@ -0,0 +1,60 @@
1
+{%- if messages[0]["role"] == "system" %}
2
+ {%- set system_message = messages[0]["content"] %}
3
+ {%- set loop_messages = messages[1:] %}
4
+{%- else %}
5
+ {%- set loop_messages = messages %}
6
+{%- endif %}
7
+
8
+{%- if not tools is defined %}
9
+ {%- set tools = none %}
10
+{%- endif %}
11
+
12
+{{- bos_token }}
13
+{%- if system_message is defined %}
14
+{{- "<|im_start|>system\n" + system_message + "<|im_end|>\n" }}
15
+{%- endif %}
16
+
17
+{%- if tools is not none %}
18
+ {{- "<|im_start|>system name=<|plugin|>\n[" }}
19
+ {%- for tool in tools %}
20
+ {{- tool.function|tojson }}
21
+ {%- if not loop.last %}
22
+ {{- ", " }}
23
+ {%- else %}
24
+ {{- "]" }}
25
+ {%- endif %}
26
+ {%- endfor %}
27
+ {{- "<|im_end|>\n" }}
28
+{%- endif %}
29
+
30
+{%- for message in loop_messages %}
31
+ {%- if message["role"] == "user" %}
32
+ {{- "<|im_start|>user\n" + message["content"] + "<|im_end|>\n"}}
33
+ {%- elif message.tool_calls is defined and message.tool_calls is not none %}
34
+ {%- set content = message["content"] if message["content"] else "" %}
35
+ {{- "<|im_start|>assistant\n" + content }}
36
+ {%- for tool_call in message.tool_calls %}
37
+ {%- set function=tool_call.function %}
38
+ {{- "<|action_start|><|plugin|>\n" }}
39
+ {{- '{"name": "' + function.name + '", '}}
40
+ {{- '"arguments": ' + function.arguments|tojson + '}' }}
41
+ {{- "<|action_end|>" }}
42
+ {%- endfor %}
43
+ {{- "<|im_end|>\n" }}
44
+ {%- elif message["role"] == "assistant" %}
45
+ {{- "<|im_start|>assistant\n" + message["content"] + "<|im_end|>\n"}}
46
+ {%- elif message["role"] == "tool_results" or message["role"] == "tool" or message["role"] == "function" %}
47
+ {%- if message.content is defined and message.content.content is defined %}
48
+ {%- set content = message.content.content %}
49
+ {%- else %}
50
+ {%- set content = message.content %}
51
+ {%- endif %}
52
+ {{- "<|im_start|>environment name=<|plugin|>\n" + content|string + "<|im_end|>\n" }}
53
+ {%- else %}
54
+ {{- raise_exception("Only user and assistant and tool_results and tool and function roles are supported, with the exception of an initial optional system message!") }}
55
+ {%- endif %}
56
+{%- endfor %}
57
+
58
+{%- if add_generation_prompt %}
59
+{{- '<|im_start|>assistant\n' }}
60
+{%- endif %}
\ No newline at end of file
tests/tool_use/utils.py
+13
-1
@@ -87,6 +87,18 @@ CONFIGS: Dict[str, ServerConfig] = {
87
"call the tool. Otherwise, answer the user's query directly "
88
"without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "
89
"to the user's question - just respond to it normally."
90
+ },
91
+ "internlm": {
92
+ "model":
93
+ "internlm/internlm2_5-7b-chat",
94
+ "arguments": [
95
+ "--tool-call-parser", "internlm", "--chat-template",
96
+ str(VLLM_PATH /
97
+ "examples/tool_chat_template_internlm2_tool.jinja"),
98
+ "--trust_remote_code"
99
+ ],
100
+ "supports_parallel":
101
+ False,
102
}
103
}
104
@@ -109,7 +121,7 @@ WEATHER_TOOL: ChatCompletionToolParam = {
121
"type":
122
"string",
123
"description":
112
- "the two-letter abbreviation for the state "
124
+ "must the two-letter abbreviation for the state "
125
"that the city is in, e.g. 'CA' which would "
126
"mean 'California'"
127
},
vllm/entrypoints/openai/api_server.py
+10
@@ -53,6 +53,7 @@ from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
53
from vllm.entrypoints.openai.serving_engine import BaseModelPath
54
from vllm.entrypoints.openai.serving_tokenization import (
55
OpenAIServingTokenization)
56
+from vllm.entrypoints.openai.tool_parsers import ToolParserManager
57
from vllm.logger import init_logger
58
from vllm.usage.usage_lib import UsageContext
59
from vllm.utils import FlexibleArgumentParser, get_open_zmq_ipc_path
@@ -526,6 +527,15 @@ async def run_server(args, **uvicorn_kwargs) -> None:
527
logger.info("vLLM API server version %s", VLLM_VERSION)
528
logger.info("args: %s", args)
529
530
+ if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
531
+ ToolParserManager.import_tool_parser(args.tool_parser_plugin)
532
+
533
+ valide_tool_parses = ToolParserManager.tool_parsers.keys()
534
+ if args.enable_auto_tool_choice \
535
+ and args.tool_call_parser not in valide_tool_parses:
536
+ raise KeyError(f"invalid tool call parser: {args.tool_call_parser} "
537
+ f"(chose from {{ {','.join(valide_tool_parses)} }})")
538
+
539
temp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
540
temp_socket.bind(("", args.port))
541
vllm/entrypoints/openai/cli_args.py
+13
-1
@@ -12,6 +12,7 @@ from typing import List, Optional, Sequence, Union
12
from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
13
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
14
PromptAdapterPath)
15
+from vllm.entrypoints.openai.tool_parsers import ToolParserManager
16
from vllm.utils import FlexibleArgumentParser
17
18
@@ -190,16 +191,27 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
191
"Enable auto tool choice for supported models. Use --tool-call-parser"
192
"to specify which parser to use")
193
194
+ valid_tool_parsers = ToolParserManager.tool_parsers.keys()
195
parser.add_argument(
196
"--tool-call-parser",
197
type=str,
196
- choices=["mistral", "hermes", "llama3_json"],
198
+ metavar="{" + ",".join(valid_tool_parsers) + "} or name registered in "
199
+ "--tool-parser-plugin",
200
default=None,
201
help=
202
"Select the tool call parser depending on the model that you're using."
203
" This is used to parse the model-generated tool call into OpenAI API "
204
"format. Required for --enable-auto-tool-choice.")
205
206
+ parser.add_argument(
207
+ "--tool-parser-plugin",
208
+ type=str,
209
+ default="",
210
+ help=
211
+ "Special the tool parser plugin write to parse the model-generated tool"
212
+ " into OpenAI API format, the name register in this plugin can be used "
213
+ "in --tool-call-parser.")
214
+
215
parser = AsyncEngineArgs.add_cli_args(parser)
216
217
parser.add_argument('--max-log-len',
vllm/entrypoints/openai/serving_chat.py
+20
-18
@@ -29,10 +29,7 @@ from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
29
OpenAIServing,
30
PromptAdapterPath,
31
TextTokensPrompt)
32
-from vllm.entrypoints.openai.tool_parsers import (Hermes2ProToolParser,
33
- Llama3JsonToolParser,
34
- MistralToolParser,
35
- ToolParser)
32
+from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
33
from vllm.inputs import TokensPrompt
34
from vllm.logger import init_logger
35
from vllm.outputs import CompletionOutput, RequestOutput
@@ -82,15 +79,13 @@ class OpenAIServingChat(OpenAIServing):
79
80
self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
81
if self.enable_auto_tools:
85
- if tool_parser == "mistral":
86
- self.tool_parser = MistralToolParser
87
- elif tool_parser == "hermes":
88
- self.tool_parser = Hermes2ProToolParser
89
- elif tool_parser == "llama3_json":
90
- self.tool_parser = Llama3JsonToolParser
91
- else:
82
+ try:
83
+ self.tool_parser = ToolParserManager.get_tool_parser(
84
+ tool_parser)
85
+ except Exception as e:
86
raise TypeError("Error: --enable-auto-tool-choice requires "
93
- "--tool-call-parser")
87
+ f"tool_parser:'{tool_parser}' which has not "
88
+ "been registered") from e
89
90
async def create_chat_completion(
91
self,
@@ -187,6 +182,10 @@ class OpenAIServingChat(OpenAIServing):
182
raw_request.state.request_metadata = request_metadata
183
184
try:
185
+ if self.enable_auto_tools and self.tool_parser:
186
+ request = self.tool_parser(tokenizer).adjust_request(
187
+ request=request)
188
+
189
if isinstance(prompt, str):
190
prompt_inputs = self._tokenize_prompt_input(
191
request,
@@ -282,11 +281,11 @@ class OpenAIServingChat(OpenAIServing):
281
num_choices = 1 if request.n is None else request.n
282
previous_num_tokens = [0] * num_choices
283
finish_reason_sent = [False] * num_choices
285
-
284
num_prompt_tokens = 0
285
288
- tool_parser: Optional[ToolParser] = self.tool_parser(
289
- tokenizer) if self.tool_parser else None
286
+ tool_parsers: List[Optional[ToolParser]] = [
287
+ self.tool_parser(tokenizer) if self.tool_parser else None
288
+ ] * num_choices
289
290
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
291
tool_choice_function_name = request.tool_choice.function.name
@@ -324,7 +323,7 @@ class OpenAIServingChat(OpenAIServing):
323
# NOTE num_choices defaults to 1 so this usually executes
324
# once per request
325
for i in range(num_choices):
327
-
326
+ tool_parser = tool_parsers[i]
327
choice_data = ChatCompletionResponseStreamChoice(
328
index=i,
329
delta=DeltaMessage(
@@ -399,6 +398,7 @@ class OpenAIServingChat(OpenAIServing):
398
399
for output in res.outputs:
400
i = output.index
401
+ tool_parser = tool_parsers[i]
402
403
if finish_reason_sent[i]:
404
continue
@@ -446,7 +446,8 @@ class OpenAIServingChat(OpenAIServing):
446
delta_text=delta_text,
447
previous_token_ids=previous_token_ids,
448
current_token_ids=current_token_ids,
449
- delta_token_ids=output.token_ids))
449
+ delta_token_ids=output.token_ids,
450
+ request=request))
451
452
# update the previous values for the next iteration
453
previous_texts[i] = current_text
@@ -685,7 +686,8 @@ class OpenAIServingChat(OpenAIServing):
686
and self.tool_parser:
687
688
tool_parser = self.tool_parser(tokenizer)
688
- tool_call_info = tool_parser.extract_tool_calls(output.text)
689
+ tool_call_info = tool_parser.extract_tool_calls(
690
+ output.text, request=request)
691
tools_called = tool_call_info.tools_called
692
if tool_call_info.tools_called:
693
message = ChatMessage(role=role,
vllm/entrypoints/openai/tool_parsers/__init__.py
+4
-3
@@ -1,9 +1,10 @@
1
-from .abstract_tool_parser import ToolParser
1
+from .abstract_tool_parser import ToolParser, ToolParserManager
2
from .hermes_tool_parser import Hermes2ProToolParser
3
+from .internlm2_tool_parser import Internlm2ToolParser
4
from .llama_tool_parser import Llama3JsonToolParser
5
from .mistral_tool_parser import MistralToolParser
6
7
__all__ = [
7
- "ToolParser", "Hermes2ProToolParser", "MistralToolParser",
8
- "Llama3JsonToolParser"
8
+ "ToolParser", "ToolParserManager", "Hermes2ProToolParser",
9
+ "MistralToolParser", "Internlm2ToolParser", "Llama3JsonToolParser"
10
]
vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py
+101
-4
@@ -1,9 +1,14 @@
1
-from typing import Dict, List, Sequence, Union
1
+import importlib
2
+import importlib.util
3
+import os
4
+from typing import Callable, Dict, List, Optional, Sequence, Type, Union
5
3
-from vllm.entrypoints.openai.protocol import (DeltaMessage,
6
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
7
+ DeltaMessage,
8
ExtractedToolCallInformation)
9
from vllm.logger import init_logger
10
from vllm.transformers_utils.tokenizer import AnyTokenizer
11
+from vllm.utils import is_list_of
12
13
logger = init_logger(__name__)
14
@@ -24,8 +29,16 @@ class ToolParser:
29
30
self.model_tokenizer = tokenizer
31
27
- def extract_tool_calls(self,
28
- model_output: str) -> ExtractedToolCallInformation:
32
+ def adjust_request(
33
+ self, request: ChatCompletionRequest) -> ChatCompletionRequest:
34
+ """
35
+ Static method that used to adjust the request parameters.
36
+ """
37
+ return request
38
+
39
+ def extract_tool_calls(
40
+ self, model_output: str,
41
+ request: ChatCompletionRequest) -> ExtractedToolCallInformation:
42
"""
43
Static method that should be implemented for extracting tool calls from
44
a complete model-generated string.
@@ -44,6 +57,7 @@ class ToolParser:
57
previous_token_ids: Sequence[int],
58
current_token_ids: Sequence[int],
59
delta_token_ids: Sequence[int],
60
+ request: ChatCompletionRequest,
61
) -> Union[DeltaMessage, None]:
62
"""
63
Instance method that should be implemented for extracting tool calls
@@ -55,3 +69,86 @@ class ToolParser:
69
raise NotImplementedError(
70
"AbstractToolParser.extract_tool_calls_streaming has not been "
71
"implemented!")
72
+
73
+
74
+class ToolParserManager:
75
+ tool_parsers: Dict[str, Type] = {}
76
+
77
+ @classmethod
78
+ def get_tool_parser(cls, name) -> Type:
79
+ """
80
+ Get tool parser by name which is registered by `register_module`.
81
+
82
+ Raise a KeyError exception if the name is not registered.
83
+ """
84
+ if name in cls.tool_parsers:
85
+ return cls.tool_parsers[name]
86
+
87
+ raise KeyError(f"tool helper: '{name}' not found in tool_parsers")
88
+
89
+ @classmethod
90
+ def _register_module(cls,
91
+ module: Type,
92
+ module_name: Optional[Union[str, List[str]]] = None,
93
+ force: bool = True) -> None:
94
+ if not issubclass(module, ToolParser):
95
+ raise TypeError(
96
+ f'module must be subclass of ToolParser, but got {type(module)}'
97
+ )
98
+ if module_name is None:
99
+ module_name = module.__name__
100
+ if isinstance(module_name, str):
101
+ module_name = [module_name]
102
+ for name in module_name:
103
+ if not force and name in cls.tool_parsers:
104
+ existed_module = cls.tool_parsers[name]
105
+ raise KeyError(f'{name} is already registered '
106
+ f'at {existed_module.__module__}')
107
+ cls.tool_parsers[name] = module
108
+
109
+ @classmethod
110
+ def register_module(
111
+ cls,
112
+ name: Optional[Union[str, List[str]]] = None,
113
+ force: bool = True,
114
+ module: Union[Type, None] = None) -> Union[type, Callable]:
115
+ """
116
+ Register module with the given name or name list. it can be used as a
117
+ decoder(with module as None) or normal function(with module as not
118
+ None).
119
+ """
120
+ if not isinstance(force, bool):
121
+ raise TypeError(f'force must be a boolean, but got {type(force)}')
122
+
123
+ # raise the error ahead of time
124
+ if not (name is None or isinstance(name, str)
125
+ or is_list_of(name, str)):
126
+ raise TypeError(
127
+ 'name must be None, an instance of str, or a sequence of str, '
128
+ f'but got {type(name)}')
129
+
130
+ # use it as a normal method: x.register_module(module=SomeClass)
131
+ if module is not None:
132
+ cls._register_module(module=module, module_name=name, force=force)
133
+ return module
134
+
135
+ # use it as a decorator: @x.register_module()
136
+ def _register(module):
137
+ cls._register_module(module=module, module_name=name, force=force)
138
+ return module
139
+
140
+ return _register
141
+
142
+ @classmethod
143
+ def import_tool_parser(cls, plugin_path: str) -> None:
144
+ """
145
+ Import a user defined tool parser by the path of the tool parser define
146
+ file.
147
+ """
148
+ module_name = os.path.splitext(os.path.basename(plugin_path))[0]
149
+ spec = importlib.util.spec_from_file_location(module_name, plugin_path)
150
+ if spec is None or spec.loader is None:
151
+ logger.error("load %s from %s failed.", module_name, plugin_path)
152
+ return
153
+ module = importlib.util.module_from_spec(spec)
154
+ spec.loader.exec_module(module)
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+10
-4
@@ -5,12 +5,13 @@ from typing import Dict, List, Sequence, Union
5
import partial_json_parser
6
from partial_json_parser.core.options import Allow
7
8
-from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
8
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
9
+ DeltaFunctionCall, DeltaMessage,
10
DeltaToolCall,
11
ExtractedToolCallInformation,
12
FunctionCall, ToolCall)
13
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
13
- ToolParser)
14
+ ToolParser, ToolParserManager)
15
from vllm.entrypoints.openai.tool_parsers.utils import (
16
extract_intermediate_diff)
17
from vllm.logger import init_logger
@@ -20,6 +21,7 @@ from vllm.utils import random_uuid
21
logger = init_logger(__name__)
22
23
24
+@ToolParserManager.register_module("hermes")
25
class Hermes2ProToolParser(ToolParser):
26
27
def __init__(self, tokenizer: AnyTokenizer):
@@ -57,8 +59,11 @@ class Hermes2ProToolParser(ToolParser):
59
"Hermes 2 Pro Tool parser could not locate tool call start/end "
60
"tokens in the tokenizer!")
61
60
- def extract_tool_calls(self,
61
- model_output: str) -> ExtractedToolCallInformation:
62
+ def extract_tool_calls(
63
+ self,
64
+ model_output: str,
65
+ request: ChatCompletionRequest,
66
+ ) -> ExtractedToolCallInformation:
67
68
# sanity check; avoid unnecessary processing
69
if self.tool_call_start_token not in model_output:
@@ -114,6 +119,7 @@ class Hermes2ProToolParser(ToolParser):
119
previous_token_ids: Sequence[int],
120
current_token_ids: Sequence[int],
121
delta_token_ids: Sequence[int],
122
+ request: ChatCompletionRequest,
123
) -> Union[DeltaMessage, None]:
124
125
logger.debug("delta_text: %s", delta_text)
vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py
new
+208
@@ -0,0 +1,208 @@
1
+import json
2
+from typing import Dict, Sequence, Union
3
+
4
+import partial_json_parser
5
+from partial_json_parser.core.options import Allow
6
+
7
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
8
+ DeltaFunctionCall, DeltaMessage,
9
+ DeltaToolCall,
10
+ ExtractedToolCallInformation,
11
+ FunctionCall, ToolCall)
12
+from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
13
+ ToolParser, ToolParserManager)
14
+from vllm.entrypoints.openai.tool_parsers.utils import (
15
+ extract_intermediate_diff)
16
+from vllm.logger import init_logger
17
+from vllm.transformers_utils.tokenizer import AnyTokenizer
18
+from vllm.utils import random_uuid
19
+
20
+logger = init_logger(__name__)
21
+
22
+
23
+@ToolParserManager.register_module(["internlm"])
24
+class Internlm2ToolParser(ToolParser):
25
+
26
+ def __init__(self, tokenizer: AnyTokenizer):
27
+ super().__init__(tokenizer)
28
+ self.position = 0
29
+
30
+ def adjust_request(
31
+ self, request: ChatCompletionRequest) -> ChatCompletionRequest:
32
+ if request.tools and request.tool_choice != 'none':
33
+ # do not skip special tokens because internlm use the special
34
+ # tokens to indicated the start and end of the tool calls
35
+ # information.
36
+ request.skip_special_tokens = False
37
+ return request
38
+
39
+ def get_argments(self, obj):
40
+ if "parameters" in obj:
41
+ return obj.get("parameters")
42
+ elif "arguments" in obj:
43
+ return obj.get("arguments")
44
+ return None
45
+
46
+ def extract_tool_calls_streaming(
47
+ self,
48
+ previous_text: str,
49
+ current_text: str,
50
+ delta_text: str,
51
+ previous_token_ids: Sequence[int],
52
+ current_token_ids: Sequence[int],
53
+ delta_token_ids: Sequence[int],
54
+ request: ChatCompletionRequest,
55
+ ) -> Union[DeltaMessage, None]:
56
+ if '<|action_start|>' not in current_text:
57
+ self.position = len(current_text)
58
+ return DeltaMessage(content=delta_text)
59
+ # if the tool call is sended, return a empty delta message
60
+ # to make sure the finish_reason will be send correctly.
61
+ if self.current_tool_id > 0:
62
+ return DeltaMessage(content='')
63
+
64
+ last_pos = self.position
65
+ if '<|action_start|><|plugin|>' not in current_text[last_pos:]:
66
+ return None
67
+
68
+ new_delta = current_text[last_pos:]
69
+ text, action = new_delta.split('<|action_start|><|plugin|>')
70
+
71
+ if len(text) > 0:
72
+ self.position = self.position + len(text)
73
+ return DeltaMessage(content=text)
74
+
75
+ action = action.strip()
76
+ action = action.split('<|action_end|>'.strip())[0]
77
+
78
+ # bit mask flags for partial JSON parsing. If the name hasn't been
79
+ # sent yet, don't allow sending
80
+ # an incomplete string since OpenAI only ever (as far as I have
81
+ # seen) allows sending the entire tool/ function name at once.
82
+ flags = Allow.ALL if self.current_tool_name_sent \
83
+ else Allow.ALL & ~Allow.STR
84
+
85
+ try:
86
+ parsable_arr = action
87
+
88
+ # tool calls are generated in an object in inernlm2
89
+ # it's not support parallel tool calls
90
+ try:
91
+ tool_call_arr: Dict = partial_json_parser.loads(
92
+ parsable_arr, flags)
93
+ except partial_json_parser.core.exceptions.MalformedJSON:
94
+ logger.debug('not enough tokens to parse into JSON yet')
95
+ return None
96
+
97
+ # if the current tool name hasn't been sent, send if available
98
+ # - otherwise send nothing
99
+ if not self.current_tool_name_sent:
100
+ function_name = tool_call_arr.get("name")
101
+ if function_name:
102
+ self.current_tool_id = self.current_tool_id + 1
103
+ delta = DeltaMessage(tool_calls=[
104
+ DeltaToolCall(index=self.current_tool_id,
105
+ type="function",
106
+ id=f"chatcmpl-tool-{random_uuid()}",
107
+ function=DeltaFunctionCall(
108
+ name=function_name).model_dump(
109
+ exclude_none=True))
110
+ ])
111
+ self.current_tool_name_sent = True
112
+ self.streamed_args_for_tool.append("")
113
+ else:
114
+ delta = None
115
+ # now we know we're on the same tool call and we're streaming
116
+ # arguments
117
+ else:
118
+ prev_arguments = self.get_argments(
119
+ self.prev_tool_call_arr[self.current_tool_id])
120
+ cur_arguments = self.get_argments(tool_call_arr)
121
+
122
+ # not arguments generated
123
+ if not cur_arguments and not prev_arguments:
124
+ delta = None
125
+ # will never happen
126
+ elif not cur_arguments and prev_arguments:
127
+ logger.error(
128
+ "INVARIANT - impossible to have arguments reset "
129
+ "mid-arguments")
130
+ delta = None
131
+ # first time to get parameters
132
+ elif cur_arguments and not prev_arguments:
133
+ cur_arguments_json = json.dumps(cur_arguments)
134
+
135
+ arguments_delta = cur_arguments_json[:cur_arguments_json.
136
+ index(delta_text) +
137
+ len(delta_text)]
138
+ delta = DeltaMessage(tool_calls=[
139
+ DeltaToolCall(index=self.current_tool_id,
140
+ function=DeltaFunctionCall(
141
+ arguments=arguments_delta).
142
+ model_dump(exclude_none=True))
143
+ ])
144
+ self.streamed_args_for_tool[
145
+ self.current_tool_id] += arguments_delta
146
+ # both prev and cur parameters, send the increase parameters
147
+ elif cur_arguments and prev_arguments:
148
+ cur_args_json = json.dumps(cur_arguments)
149
+ prev_args_json = json.dumps(prev_arguments)
150
+
151
+ argument_diff = extract_intermediate_diff(
152
+ cur_args_json, prev_args_json)
153
+
154
+ delta = DeltaMessage(tool_calls=[
155
+ DeltaToolCall(index=self.current_tool_id,
156
+ function=DeltaFunctionCall(
157
+ arguments=argument_diff).model_dump(
158
+ exclude_none=True))
159
+ ])
160
+ self.streamed_args_for_tool[
161
+ self.current_tool_id] += argument_diff
162
+
163
+ # check to see if the name is defined and has been sent. if so,
164
+ # stream the name - otherwise keep waiting
165
+ # finish by setting old and returning None as base case
166
+ tool_call_arr["arguments"] = self.get_argments(tool_call_arr)
167
+ self.prev_tool_call_arr = [tool_call_arr]
168
+ return delta
169
+ except Exception as e:
170
+ logger.error("Error trying to handle streaming tool call: %s", e)
171
+ logger.debug(
172
+ "Skipping chunk as a result of tool streaming extraction "
173
+ "error")
174
+ return None
175
+
176
+ def extract_tool_calls(
177
+ self,
178
+ model_output: str,
179
+ request: ChatCompletionRequest,
180
+ ) -> ExtractedToolCallInformation:
181
+ text = model_output
182
+ tools = request.tools
183
+ if '<|action_start|><|plugin|>' in text:
184
+ text, action = text.split('<|action_start|><|plugin|>')
185
+ action = action.split('<|action_end|>'.strip())[0]
186
+ action = action[action.find('{'):]
187
+ action_dict = json.loads(action)
188
+ name, parameters = action_dict['name'], json.dumps(
189
+ action_dict.get('parameters', action_dict.get('arguments',
190
+ {})))
191
+
192
+ if not tools or name not in [t.function.name for t in tools]:
193
+ ExtractedToolCallInformation(tools_called=False,
194
+ tool_calls=[],
195
+ content=text)
196
+
197
+ tool_calls = [
198
+ ToolCall(
199
+ function=FunctionCall(name=name, arguments=parameters))
200
+ ]
201
+ return ExtractedToolCallInformation(
202
+ tools_called=True,
203
+ tool_calls=tool_calls,
204
+ content=text if len(text) > 0 else None)
205
+
206
+ return ExtractedToolCallInformation(tools_called=False,
207
+ tool_calls=[],
208
+ content=text)
vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py
+8
-4
@@ -7,12 +7,13 @@ import partial_json_parser
7
from partial_json_parser.core.options import Allow
8
from transformers import PreTrainedTokenizerBase
9
10
-from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
10
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
11
+ DeltaFunctionCall, DeltaMessage,
12
DeltaToolCall,
13
ExtractedToolCallInformation,
14
FunctionCall, ToolCall)
15
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
15
- ToolParser)
16
+ ToolParser, ToolParserManager)
17
from vllm.entrypoints.openai.tool_parsers.utils import find_common_prefix
18
from vllm.logger import init_logger
19
from vllm.utils import random_uuid
@@ -41,6 +42,7 @@ def is_complete_json(input_str):
42
return False
43
44
45
+@ToolParserManager.register_module("llama3_json")
46
class Llama3JsonToolParser(ToolParser):
47
"""
48
Tool call parser for Llama 3.1 models intended for use with the
@@ -64,8 +66,9 @@ class Llama3JsonToolParser(ToolParser):
66
add_special_tokens=False)[0]
67
self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL)
68
67
- def extract_tool_calls(self,
68
- model_output: str) -> ExtractedToolCallInformation:
69
+ def extract_tool_calls(
70
+ self, model_output: str,
71
+ request: ChatCompletionRequest) -> ExtractedToolCallInformation:
72
"""
73
Extract the tool calls from a complete model response.
74
"""
@@ -125,6 +128,7 @@ class Llama3JsonToolParser(ToolParser):
128
previous_token_ids: Sequence[int],
129
current_token_ids: Sequence[int],
130
delta_token_ids: Sequence[int],
131
+ request: ChatCompletionRequest,
132
) -> Union[DeltaMessage, None]:
133
134
if not (current_text.startswith(self.bot_token)
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+12
-8
@@ -8,12 +8,13 @@ import partial_json_parser
8
from partial_json_parser.core.options import Allow
9
from pydantic import Field
10
11
-from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
11
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12
+ DeltaFunctionCall, DeltaMessage,
13
DeltaToolCall,
14
ExtractedToolCallInformation,
15
FunctionCall, ToolCall)
16
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
16
- ToolParser)
17
+ ToolParser, ToolParserManager)
18
from vllm.entrypoints.openai.tool_parsers.utils import (
19
extract_intermediate_diff)
20
from vllm.logger import init_logger
@@ -36,6 +37,7 @@ class MistralToolCall(ToolCall):
37
return "".join(choices(ALPHANUMERIC, k=9))
38
39
40
+@ToolParserManager.register_module("mistral")
41
class MistralToolParser(ToolParser):
42
"""
43
Tool call parser for Mistral 7B Instruct v0.3, intended for use with the
@@ -47,9 +49,7 @@ class MistralToolParser(ToolParser):
49
def __init__(self, tokenizer: AnyTokenizer):
50
super().__init__(tokenizer)
51
50
- if isinstance(self.model_tokenizer, MistralTokenizer):
51
- self.model_tokenizer = self.model_tokenizer.tokenizer
52
- else:
52
+ if not isinstance(self.model_tokenizer, MistralTokenizer):
53
logger.info("Non-Mistral tokenizer detected when using a Mistral "
54
"model...")
55
@@ -61,11 +61,14 @@ class MistralToolParser(ToolParser):
61
self.streamed_args_for_tool: List[str] = [
62
] # map what has been streamed for each tool so far to a list
63
self.bot_token = "[TOOL_CALLS]"
64
- self.bot_token_id = self.model_tokenizer.vocab[self.bot_token]
64
+ self.bot_token_id = self.model_tokenizer.get_vocab()[self.bot_token]
65
self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL)
66
67
- def extract_tool_calls(self,
68
- model_output: str) -> ExtractedToolCallInformation:
67
+ def extract_tool_calls(
68
+ self,
69
+ model_output: str,
70
+ request: ChatCompletionRequest,
71
+ ) -> ExtractedToolCallInformation:
72
"""
73
Extract the tool calls from a complete model response. Requires
74
find-and-replacing single quotes with double quotes for JSON parsing,
@@ -119,6 +122,7 @@ class MistralToolParser(ToolParser):
122
previous_token_ids: Sequence[int],
123
current_token_ids: Sequence[int],
124
delta_token_ids: Sequence[int],
125
+ request: ChatCompletionRequest,
126
) -> Union[DeltaMessage, None]:
127
128
# if the tool call token is not in the tokens generated so far, append