@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3dbb215b38
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Frontend][Feature] support tool calling for internlm/internlm2_5-7b-chat model (#8405)

代君 committed Oct 4, 2024 at 10:36 UTC 3dbb215b38c010c050f7fde3528fe2c6673f7a07
13 files changed +533 -46
docs/requirements-docs.txt
+2 -1
@@ -12,4 +12,5 @@ torch
12 py-cpuinfo
13 transformers
14 mistral_common >= 1.3.4
15 -openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
\ No newline at end of file
15 +openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
16 +partial-json-parser # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
\ No newline at end of file
docs/source/serving/openai_compatible_server.md
+72 -2
@@ -157,8 +157,9 @@ vLLM will use guided decoding to ensure the response matches the tool parameter
157 To enable this feature, you should set the following flags:
158 * `--enable-auto-tool-choice` -- **mandatory** Auto tool choice. tells vLLM that you want to enable the model to generate its own tool calls when it
159 deems appropriate.
160 -* `--tool-call-parser` -- select the tool parser to use - currently either `hermes`, `mistral` or `llama3_json`. Additional tool parsers
161 -will continue to be added in the future.
160 +* `--tool-call-parser` -- select the tool parser to use - currently either `hermes` or `mistral` or `llama3_json` or `internlm`. Additional tool parsers
161 +will continue to be added in the future, and also can register your own tool parsers in the `--tool-parser-plugin`.
162 +* `--tool-parser-plugin` -- **optional** tool parser plugin used to register user defined tool parsers into vllm, the registered tool parser name can be specified in `--tool-call-parser`.
163 * `--chat-template` -- **optional** for auto tool choice. the path to the chat template which handles `tool`-role messages and `assistant`-role messages
164 that contain previously generated tool calls. Hermes, Mistral and Llama models have tool-compatible chat templates in their
165 `tokenizer_config.json` files, but you can specify a custom template. This argument can be set to `tool_use` if your model has a tool use-specific chat
@@ -218,4 +219,73 @@ it works better with vLLM.
219
220 Recommended flags: `--tool-call-parser llama3_json --chat-template examples/tool_chat_template_llama3_json.jinja`
221
222 +#### Internlm Models
223 +Supported models:
224 +* `internlm/internlm2_5-7b-chat` (confirmed)
225 +* Additional internlm2.5 function-calling models are compatible as well
226 +
227 +Known issues:
228 +* Although this implementation also supports Internlm2, the tool call results are not stable when testing with the `internlm/internlm2-chat-7b` model.
229 +
230 +Recommended flags: `--tool-call-parser internlm --chat-template examples/tool_chat_template_internlm2_tool.jinja`
231 +
232 +
233 +### How to write a tool parser plugin
234 +
235 +A tool parser plugin is a Python file containing one or more ToolParser implementations. You can write a ToolParser similar to the `Hermes2ProToolParser` in vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py.
236 +
237 +Here is a summary of a plugin file:
238 +
239 +```python
240 +
241 +# import the required packages
242 +
243 +# define a tool parser and register it to vllm
244 +# the name list in register_module can be used
245 +# in --tool-call-parser. you can define as many
246 +# tool parsers as you want here.
247 +@ToolParserManager.register_module(["example"])
248 +class ExampleToolParser(ToolParser):
249 + def __init__(self, tokenizer: AnyTokenizer):
250 + super().__init__(tokenizer)
251 +
252 + # adjust request. e.g.: set skip special tokens
253 + # to False for tool call output.
254 + def adjust_request(
255 + self, request: ChatCompletionRequest) -> ChatCompletionRequest:
256 + return request
257 +
258 + # implement the tool call parse for stream call
259 + def extract_tool_calls_streaming(
260 + self,
261 + previous_text: str,
262 + current_text: str,
263 + delta_text: str,
264 + previous_token_ids: Sequence[int],
265 + current_token_ids: Sequence[int],
266 + delta_token_ids: Sequence[int],
267 + request: ChatCompletionRequest,
268 + ) -> Union[DeltaMessage, None]:
269 + return delta
270 +
271 + # implement the tool parse for non-stream call
272 + def extract_tool_calls(
273 + self,
274 + model_output: str,
275 + request: ChatCompletionRequest,
276 + ) -> ExtractedToolCallInformation:
277 + return ExtractedToolCallInformation(tools_called=False,
278 + tool_calls=[],
279 + content=text)
280 +
281 +
282 +```
283 +
284 +Then you can use this plugin in the command line like this.
285 +```
286 + --enable-auto-tool-choice \
287 + --tool-parser-plugin <absolute path of the plugin file>
288 + --tool-call-parser example \
289 + --chat-template <your chat template> \
290 +```
291
examples/tool_chat_template_internlm2_tool.jinja new
+60
@@ -0,0 +1,60 @@
1 +{%- if messages[0]["role"] == "system" %}
2 + {%- set system_message = messages[0]["content"] %}
3 + {%- set loop_messages = messages[1:] %}
4 +{%- else %}
5 + {%- set loop_messages = messages %}
6 +{%- endif %}
7 +
8 +{%- if not tools is defined %}
9 + {%- set tools = none %}
10 +{%- endif %}
11 +
12 +{{- bos_token }}
13 +{%- if system_message is defined %}
14 +{{- "<|im_start|>system\n" + system_message + "<|im_end|>\n" }}
15 +{%- endif %}
16 +
17 +{%- if tools is not none %}
18 + {{- "<|im_start|>system name=<|plugin|>\n[" }}
19 + {%- for tool in tools %}
20 + {{- tool.function|tojson }}
21 + {%- if not loop.last %}
22 + {{- ", " }}
23 + {%- else %}
24 + {{- "]" }}
25 + {%- endif %}
26 + {%- endfor %}
27 + {{- "<|im_end|>\n" }}
28 +{%- endif %}
29 +
30 +{%- for message in loop_messages %}
31 + {%- if message["role"] == "user" %}
32 + {{- "<|im_start|>user\n" + message["content"] + "<|im_end|>\n"}}
33 + {%- elif message.tool_calls is defined and message.tool_calls is not none %}
34 + {%- set content = message["content"] if message["content"] else "" %}
35 + {{- "<|im_start|>assistant\n" + content }}
36 + {%- for tool_call in message.tool_calls %}
37 + {%- set function=tool_call.function %}
38 + {{- "<|action_start|><|plugin|>\n" }}
39 + {{- '{"name": "' + function.name + '", '}}
40 + {{- '"arguments": ' + function.arguments|tojson + '}' }}
41 + {{- "<|action_end|>" }}
42 + {%- endfor %}
43 + {{- "<|im_end|>\n" }}
44 + {%- elif message["role"] == "assistant" %}
45 + {{- "<|im_start|>assistant\n" + message["content"] + "<|im_end|>\n"}}
46 + {%- elif message["role"] == "tool_results" or message["role"] == "tool" or message["role"] == "function" %}
47 + {%- if message.content is defined and message.content.content is defined %}
48 + {%- set content = message.content.content %}
49 + {%- else %}
50 + {%- set content = message.content %}
51 + {%- endif %}
52 + {{- "<|im_start|>environment name=<|plugin|>\n" + content|string + "<|im_end|>\n" }}
53 + {%- else %}
54 + {{- raise_exception("Only user and assistant and tool_results and tool and function roles are supported, with the exception of an initial optional system message!") }}
55 + {%- endif %}
56 +{%- endfor %}
57 +
58 +{%- if add_generation_prompt %}
59 +{{- '<|im_start|>assistant\n' }}
60 +{%- endif %}
\ No newline at end of file
tests/tool_use/utils.py
+13 -1
@@ -87,6 +87,18 @@ CONFIGS: Dict[str, ServerConfig] = {
87 "call the tool. Otherwise, answer the user's query directly "
88 "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "
89 "to the user's question - just respond to it normally."
90 + },
91 + "internlm": {
92 + "model":
93 + "internlm/internlm2_5-7b-chat",
94 + "arguments": [
95 + "--tool-call-parser", "internlm", "--chat-template",
96 + str(VLLM_PATH /
97 + "examples/tool_chat_template_internlm2_tool.jinja"),
98 + "--trust_remote_code"
99 + ],
100 + "supports_parallel":
101 + False,
102 }
103 }
104
@@ -109,7 +121,7 @@ WEATHER_TOOL: ChatCompletionToolParam = {
121 "type":
122 "string",
123 "description":
112 - "the two-letter abbreviation for the state "
124 + "must the two-letter abbreviation for the state "
125 "that the city is in, e.g. 'CA' which would "
126 "mean 'California'"
127 },
vllm/entrypoints/openai/api_server.py
+10
@@ -53,6 +53,7 @@ from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
53 from vllm.entrypoints.openai.serving_engine import BaseModelPath
54 from vllm.entrypoints.openai.serving_tokenization import (
55 OpenAIServingTokenization)
56 +from vllm.entrypoints.openai.tool_parsers import ToolParserManager
57 from vllm.logger import init_logger
58 from vllm.usage.usage_lib import UsageContext
59 from vllm.utils import FlexibleArgumentParser, get_open_zmq_ipc_path
@@ -526,6 +527,15 @@ async def run_server(args, **uvicorn_kwargs) -> None:
527 logger.info("vLLM API server version %s", VLLM_VERSION)
528 logger.info("args: %s", args)
529
530 + if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
531 + ToolParserManager.import_tool_parser(args.tool_parser_plugin)
532 +
533 + valide_tool_parses = ToolParserManager.tool_parsers.keys()
534 + if args.enable_auto_tool_choice \
535 + and args.tool_call_parser not in valide_tool_parses:
536 + raise KeyError(f"invalid tool call parser: {args.tool_call_parser} "
537 + f"(chose from {{ {','.join(valide_tool_parses)} }})")
538 +
539 temp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
540 temp_socket.bind(("", args.port))
541
vllm/entrypoints/openai/cli_args.py
+13 -1
@@ -12,6 +12,7 @@ from typing import List, Optional, Sequence, Union
12 from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
13 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
14 PromptAdapterPath)
15 +from vllm.entrypoints.openai.tool_parsers import ToolParserManager
16 from vllm.utils import FlexibleArgumentParser
17
18
@@ -190,16 +191,27 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
191 "Enable auto tool choice for supported models. Use --tool-call-parser"
192 "to specify which parser to use")
193
194 + valid_tool_parsers = ToolParserManager.tool_parsers.keys()
195 parser.add_argument(
196 "--tool-call-parser",
197 type=str,
196 - choices=["mistral", "hermes", "llama3_json"],
198 + metavar="{" + ",".join(valid_tool_parsers) + "} or name registered in "
199 + "--tool-parser-plugin",
200 default=None,
201 help=
202 "Select the tool call parser depending on the model that you're using."
203 " This is used to parse the model-generated tool call into OpenAI API "
204 "format. Required for --enable-auto-tool-choice.")
205
206 + parser.add_argument(
207 + "--tool-parser-plugin",
208 + type=str,
209 + default="",
210 + help=
211 + "Special the tool parser plugin write to parse the model-generated tool"
212 + " into OpenAI API format, the name register in this plugin can be used "
213 + "in --tool-call-parser.")
214 +
215 parser = AsyncEngineArgs.add_cli_args(parser)
216
217 parser.add_argument('--max-log-len',
vllm/entrypoints/openai/serving_chat.py
+20 -18
@@ -29,10 +29,7 @@ from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
29 OpenAIServing,
30 PromptAdapterPath,
31 TextTokensPrompt)
32 -from vllm.entrypoints.openai.tool_parsers import (Hermes2ProToolParser,
33 - Llama3JsonToolParser,
34 - MistralToolParser,
35 - ToolParser)
32 +from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
33 from vllm.inputs import TokensPrompt
34 from vllm.logger import init_logger
35 from vllm.outputs import CompletionOutput, RequestOutput
@@ -82,15 +79,13 @@ class OpenAIServingChat(OpenAIServing):
79
80 self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
81 if self.enable_auto_tools:
85 - if tool_parser == "mistral":
86 - self.tool_parser = MistralToolParser
87 - elif tool_parser == "hermes":
88 - self.tool_parser = Hermes2ProToolParser
89 - elif tool_parser == "llama3_json":
90 - self.tool_parser = Llama3JsonToolParser
91 - else:
82 + try:
83 + self.tool_parser = ToolParserManager.get_tool_parser(
84 + tool_parser)
85 + except Exception as e:
86 raise TypeError("Error: --enable-auto-tool-choice requires "
93 - "--tool-call-parser")
87 + f"tool_parser:'{tool_parser}' which has not "
88 + "been registered") from e
89
90 async def create_chat_completion(
91 self,
@@ -187,6 +182,10 @@ class OpenAIServingChat(OpenAIServing):
182 raw_request.state.request_metadata = request_metadata
183
184 try:
185 + if self.enable_auto_tools and self.tool_parser:
186 + request = self.tool_parser(tokenizer).adjust_request(
187 + request=request)
188 +
189 if isinstance(prompt, str):
190 prompt_inputs = self._tokenize_prompt_input(
191 request,
@@ -282,11 +281,11 @@ class OpenAIServingChat(OpenAIServing):
281 num_choices = 1 if request.n is None else request.n
282 previous_num_tokens = [0] * num_choices
283 finish_reason_sent = [False] * num_choices
285 -
284 num_prompt_tokens = 0
285
288 - tool_parser: Optional[ToolParser] = self.tool_parser(
289 - tokenizer) if self.tool_parser else None
286 + tool_parsers: List[Optional[ToolParser]] = [
287 + self.tool_parser(tokenizer) if self.tool_parser else None
288 + ] * num_choices
289
290 if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
291 tool_choice_function_name = request.tool_choice.function.name
@@ -324,7 +323,7 @@ class OpenAIServingChat(OpenAIServing):
323 # NOTE num_choices defaults to 1 so this usually executes
324 # once per request
325 for i in range(num_choices):
327 -
326 + tool_parser = tool_parsers[i]
327 choice_data = ChatCompletionResponseStreamChoice(
328 index=i,
329 delta=DeltaMessage(
@@ -399,6 +398,7 @@ class OpenAIServingChat(OpenAIServing):
398
399 for output in res.outputs:
400 i = output.index
401 + tool_parser = tool_parsers[i]
402
403 if finish_reason_sent[i]:
404 continue
@@ -446,7 +446,8 @@ class OpenAIServingChat(OpenAIServing):
446 delta_text=delta_text,
447 previous_token_ids=previous_token_ids,
448 current_token_ids=current_token_ids,
449 - delta_token_ids=output.token_ids))
449 + delta_token_ids=output.token_ids,
450 + request=request))
451
452 # update the previous values for the next iteration
453 previous_texts[i] = current_text
@@ -685,7 +686,8 @@ class OpenAIServingChat(OpenAIServing):
686 and self.tool_parser:
687
688 tool_parser = self.tool_parser(tokenizer)
688 - tool_call_info = tool_parser.extract_tool_calls(output.text)
689 + tool_call_info = tool_parser.extract_tool_calls(
690 + output.text, request=request)
691 tools_called = tool_call_info.tools_called
692 if tool_call_info.tools_called:
693 message = ChatMessage(role=role,
vllm/entrypoints/openai/tool_parsers/__init__.py
+4 -3
@@ -1,9 +1,10 @@
1 -from .abstract_tool_parser import ToolParser
1 +from .abstract_tool_parser import ToolParser, ToolParserManager
2 from .hermes_tool_parser import Hermes2ProToolParser
3 +from .internlm2_tool_parser import Internlm2ToolParser
4 from .llama_tool_parser import Llama3JsonToolParser
5 from .mistral_tool_parser import MistralToolParser
6
7 __all__ = [
7 - "ToolParser", "Hermes2ProToolParser", "MistralToolParser",
8 - "Llama3JsonToolParser"
8 + "ToolParser", "ToolParserManager", "Hermes2ProToolParser",
9 + "MistralToolParser", "Internlm2ToolParser", "Llama3JsonToolParser"
10 ]
vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py
+101 -4
@@ -1,9 +1,14 @@
1 -from typing import Dict, List, Sequence, Union
1 +import importlib
2 +import importlib.util
3 +import os
4 +from typing import Callable, Dict, List, Optional, Sequence, Type, Union
5
3 -from vllm.entrypoints.openai.protocol import (DeltaMessage,
6 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
7 + DeltaMessage,
8 ExtractedToolCallInformation)
9 from vllm.logger import init_logger
10 from vllm.transformers_utils.tokenizer import AnyTokenizer
11 +from vllm.utils import is_list_of
12
13 logger = init_logger(__name__)
14
@@ -24,8 +29,16 @@ class ToolParser:
29
30 self.model_tokenizer = tokenizer
31
27 - def extract_tool_calls(self,
28 - model_output: str) -> ExtractedToolCallInformation:
32 + def adjust_request(
33 + self, request: ChatCompletionRequest) -> ChatCompletionRequest:
34 + """
35 + Static method that used to adjust the request parameters.
36 + """
37 + return request
38 +
39 + def extract_tool_calls(
40 + self, model_output: str,
41 + request: ChatCompletionRequest) -> ExtractedToolCallInformation:
42 """
43 Static method that should be implemented for extracting tool calls from
44 a complete model-generated string.
@@ -44,6 +57,7 @@ class ToolParser:
57 previous_token_ids: Sequence[int],
58 current_token_ids: Sequence[int],
59 delta_token_ids: Sequence[int],
60 + request: ChatCompletionRequest,
61 ) -> Union[DeltaMessage, None]:
62 """
63 Instance method that should be implemented for extracting tool calls
@@ -55,3 +69,86 @@ class ToolParser:
69 raise NotImplementedError(
70 "AbstractToolParser.extract_tool_calls_streaming has not been "
71 "implemented!")
72 +
73 +
74 +class ToolParserManager:
75 + tool_parsers: Dict[str, Type] = {}
76 +
77 + @classmethod
78 + def get_tool_parser(cls, name) -> Type:
79 + """
80 + Get tool parser by name which is registered by `register_module`.
81 +
82 + Raise a KeyError exception if the name is not registered.
83 + """
84 + if name in cls.tool_parsers:
85 + return cls.tool_parsers[name]
86 +
87 + raise KeyError(f"tool helper: '{name}' not found in tool_parsers")
88 +
89 + @classmethod
90 + def _register_module(cls,
91 + module: Type,
92 + module_name: Optional[Union[str, List[str]]] = None,
93 + force: bool = True) -> None:
94 + if not issubclass(module, ToolParser):
95 + raise TypeError(
96 + f'module must be subclass of ToolParser, but got {type(module)}'
97 + )
98 + if module_name is None:
99 + module_name = module.__name__
100 + if isinstance(module_name, str):
101 + module_name = [module_name]
102 + for name in module_name:
103 + if not force and name in cls.tool_parsers:
104 + existed_module = cls.tool_parsers[name]
105 + raise KeyError(f'{name} is already registered '
106 + f'at {existed_module.__module__}')
107 + cls.tool_parsers[name] = module
108 +
109 + @classmethod
110 + def register_module(
111 + cls,
112 + name: Optional[Union[str, List[str]]] = None,
113 + force: bool = True,
114 + module: Union[Type, None] = None) -> Union[type, Callable]:
115 + """
116 + Register module with the given name or name list. it can be used as a
117 + decoder(with module as None) or normal function(with module as not
118 + None).
119 + """
120 + if not isinstance(force, bool):
121 + raise TypeError(f'force must be a boolean, but got {type(force)}')
122 +
123 + # raise the error ahead of time
124 + if not (name is None or isinstance(name, str)
125 + or is_list_of(name, str)):
126 + raise TypeError(
127 + 'name must be None, an instance of str, or a sequence of str, '
128 + f'but got {type(name)}')
129 +
130 + # use it as a normal method: x.register_module(module=SomeClass)
131 + if module is not None:
132 + cls._register_module(module=module, module_name=name, force=force)
133 + return module
134 +
135 + # use it as a decorator: @x.register_module()
136 + def _register(module):
137 + cls._register_module(module=module, module_name=name, force=force)
138 + return module
139 +
140 + return _register
141 +
142 + @classmethod
143 + def import_tool_parser(cls, plugin_path: str) -> None:
144 + """
145 + Import a user defined tool parser by the path of the tool parser define
146 + file.
147 + """
148 + module_name = os.path.splitext(os.path.basename(plugin_path))[0]
149 + spec = importlib.util.spec_from_file_location(module_name, plugin_path)
150 + if spec is None or spec.loader is None:
151 + logger.error("load %s from %s failed.", module_name, plugin_path)
152 + return
153 + module = importlib.util.module_from_spec(spec)
154 + spec.loader.exec_module(module)
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+10 -4
@@ -5,12 +5,13 @@ from typing import Dict, List, Sequence, Union
5 import partial_json_parser
6 from partial_json_parser.core.options import Allow
7
8 -from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
8 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
9 + DeltaFunctionCall, DeltaMessage,
10 DeltaToolCall,
11 ExtractedToolCallInformation,
12 FunctionCall, ToolCall)
13 from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
13 - ToolParser)
14 + ToolParser, ToolParserManager)
15 from vllm.entrypoints.openai.tool_parsers.utils import (
16 extract_intermediate_diff)
17 from vllm.logger import init_logger
@@ -20,6 +21,7 @@ from vllm.utils import random_uuid
21 logger = init_logger(__name__)
22
23
24 +@ToolParserManager.register_module("hermes")
25 class Hermes2ProToolParser(ToolParser):
26
27 def __init__(self, tokenizer: AnyTokenizer):
@@ -57,8 +59,11 @@ class Hermes2ProToolParser(ToolParser):
59 "Hermes 2 Pro Tool parser could not locate tool call start/end "
60 "tokens in the tokenizer!")
61
60 - def extract_tool_calls(self,
61 - model_output: str) -> ExtractedToolCallInformation:
62 + def extract_tool_calls(
63 + self,
64 + model_output: str,
65 + request: ChatCompletionRequest,
66 + ) -> ExtractedToolCallInformation:
67
68 # sanity check; avoid unnecessary processing
69 if self.tool_call_start_token not in model_output:
@@ -114,6 +119,7 @@ class Hermes2ProToolParser(ToolParser):
119 previous_token_ids: Sequence[int],
120 current_token_ids: Sequence[int],
121 delta_token_ids: Sequence[int],
122 + request: ChatCompletionRequest,
123 ) -> Union[DeltaMessage, None]:
124
125 logger.debug("delta_text: %s", delta_text)
vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py new
+208
@@ -0,0 +1,208 @@
1 +import json
2 +from typing import Dict, Sequence, Union
3 +
4 +import partial_json_parser
5 +from partial_json_parser.core.options import Allow
6 +
7 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
8 + DeltaFunctionCall, DeltaMessage,
9 + DeltaToolCall,
10 + ExtractedToolCallInformation,
11 + FunctionCall, ToolCall)
12 +from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
13 + ToolParser, ToolParserManager)
14 +from vllm.entrypoints.openai.tool_parsers.utils import (
15 + extract_intermediate_diff)
16 +from vllm.logger import init_logger
17 +from vllm.transformers_utils.tokenizer import AnyTokenizer
18 +from vllm.utils import random_uuid
19 +
20 +logger = init_logger(__name__)
21 +
22 +
23 +@ToolParserManager.register_module(["internlm"])
24 +class Internlm2ToolParser(ToolParser):
25 +
26 + def __init__(self, tokenizer: AnyTokenizer):
27 + super().__init__(tokenizer)
28 + self.position = 0
29 +
30 + def adjust_request(
31 + self, request: ChatCompletionRequest) -> ChatCompletionRequest:
32 + if request.tools and request.tool_choice != 'none':
33 + # do not skip special tokens because internlm use the special
34 + # tokens to indicated the start and end of the tool calls
35 + # information.
36 + request.skip_special_tokens = False
37 + return request
38 +
39 + def get_argments(self, obj):
40 + if "parameters" in obj:
41 + return obj.get("parameters")
42 + elif "arguments" in obj:
43 + return obj.get("arguments")
44 + return None
45 +
46 + def extract_tool_calls_streaming(
47 + self,
48 + previous_text: str,
49 + current_text: str,
50 + delta_text: str,
51 + previous_token_ids: Sequence[int],
52 + current_token_ids: Sequence[int],
53 + delta_token_ids: Sequence[int],
54 + request: ChatCompletionRequest,
55 + ) -> Union[DeltaMessage, None]:
56 + if '<|action_start|>' not in current_text:
57 + self.position = len(current_text)
58 + return DeltaMessage(content=delta_text)
59 + # if the tool call is sended, return a empty delta message
60 + # to make sure the finish_reason will be send correctly.
61 + if self.current_tool_id > 0:
62 + return DeltaMessage(content='')
63 +
64 + last_pos = self.position
65 + if '<|action_start|><|plugin|>' not in current_text[last_pos:]:
66 + return None
67 +
68 + new_delta = current_text[last_pos:]
69 + text, action = new_delta.split('<|action_start|><|plugin|>')
70 +
71 + if len(text) > 0:
72 + self.position = self.position + len(text)
73 + return DeltaMessage(content=text)
74 +
75 + action = action.strip()
76 + action = action.split('<|action_end|>'.strip())[0]
77 +
78 + # bit mask flags for partial JSON parsing. If the name hasn't been
79 + # sent yet, don't allow sending
80 + # an incomplete string since OpenAI only ever (as far as I have
81 + # seen) allows sending the entire tool/ function name at once.
82 + flags = Allow.ALL if self.current_tool_name_sent \
83 + else Allow.ALL & ~Allow.STR
84 +
85 + try:
86 + parsable_arr = action
87 +
88 + # tool calls are generated in an object in inernlm2
89 + # it's not support parallel tool calls
90 + try:
91 + tool_call_arr: Dict = partial_json_parser.loads(
92 + parsable_arr, flags)
93 + except partial_json_parser.core.exceptions.MalformedJSON:
94 + logger.debug('not enough tokens to parse into JSON yet')
95 + return None
96 +
97 + # if the current tool name hasn't been sent, send if available
98 + # - otherwise send nothing
99 + if not self.current_tool_name_sent:
100 + function_name = tool_call_arr.get("name")
101 + if function_name:
102 + self.current_tool_id = self.current_tool_id + 1
103 + delta = DeltaMessage(tool_calls=[
104 + DeltaToolCall(index=self.current_tool_id,
105 + type="function",
106 + id=f"chatcmpl-tool-{random_uuid()}",
107 + function=DeltaFunctionCall(
108 + name=function_name).model_dump(
109 + exclude_none=True))
110 + ])
111 + self.current_tool_name_sent = True
112 + self.streamed_args_for_tool.append("")
113 + else:
114 + delta = None
115 + # now we know we're on the same tool call and we're streaming
116 + # arguments
117 + else:
118 + prev_arguments = self.get_argments(
119 + self.prev_tool_call_arr[self.current_tool_id])
120 + cur_arguments = self.get_argments(tool_call_arr)
121 +
122 + # not arguments generated
123 + if not cur_arguments and not prev_arguments:
124 + delta = None
125 + # will never happen
126 + elif not cur_arguments and prev_arguments:
127 + logger.error(
128 + "INVARIANT - impossible to have arguments reset "
129 + "mid-arguments")
130 + delta = None
131 + # first time to get parameters
132 + elif cur_arguments and not prev_arguments:
133 + cur_arguments_json = json.dumps(cur_arguments)
134 +
135 + arguments_delta = cur_arguments_json[:cur_arguments_json.
136 + index(delta_text) +
137 + len(delta_text)]
138 + delta = DeltaMessage(tool_calls=[
139 + DeltaToolCall(index=self.current_tool_id,
140 + function=DeltaFunctionCall(
141 + arguments=arguments_delta).
142 + model_dump(exclude_none=True))
143 + ])
144 + self.streamed_args_for_tool[
145 + self.current_tool_id] += arguments_delta
146 + # both prev and cur parameters, send the increase parameters
147 + elif cur_arguments and prev_arguments:
148 + cur_args_json = json.dumps(cur_arguments)
149 + prev_args_json = json.dumps(prev_arguments)
150 +
151 + argument_diff = extract_intermediate_diff(
152 + cur_args_json, prev_args_json)
153 +
154 + delta = DeltaMessage(tool_calls=[
155 + DeltaToolCall(index=self.current_tool_id,
156 + function=DeltaFunctionCall(
157 + arguments=argument_diff).model_dump(
158 + exclude_none=True))
159 + ])
160 + self.streamed_args_for_tool[
161 + self.current_tool_id] += argument_diff
162 +
163 + # check to see if the name is defined and has been sent. if so,
164 + # stream the name - otherwise keep waiting
165 + # finish by setting old and returning None as base case
166 + tool_call_arr["arguments"] = self.get_argments(tool_call_arr)
167 + self.prev_tool_call_arr = [tool_call_arr]
168 + return delta
169 + except Exception as e:
170 + logger.error("Error trying to handle streaming tool call: %s", e)
171 + logger.debug(
172 + "Skipping chunk as a result of tool streaming extraction "
173 + "error")
174 + return None
175 +
176 + def extract_tool_calls(
177 + self,
178 + model_output: str,
179 + request: ChatCompletionRequest,
180 + ) -> ExtractedToolCallInformation:
181 + text = model_output
182 + tools = request.tools
183 + if '<|action_start|><|plugin|>' in text:
184 + text, action = text.split('<|action_start|><|plugin|>')
185 + action = action.split('<|action_end|>'.strip())[0]
186 + action = action[action.find('{'):]
187 + action_dict = json.loads(action)
188 + name, parameters = action_dict['name'], json.dumps(
189 + action_dict.get('parameters', action_dict.get('arguments',
190 + {})))
191 +
192 + if not tools or name not in [t.function.name for t in tools]:
193 + ExtractedToolCallInformation(tools_called=False,
194 + tool_calls=[],
195 + content=text)
196 +
197 + tool_calls = [
198 + ToolCall(
199 + function=FunctionCall(name=name, arguments=parameters))
200 + ]
201 + return ExtractedToolCallInformation(
202 + tools_called=True,
203 + tool_calls=tool_calls,
204 + content=text if len(text) > 0 else None)
205 +
206 + return ExtractedToolCallInformation(tools_called=False,
207 + tool_calls=[],
208 + content=text)
vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py
+8 -4
@@ -7,12 +7,13 @@ import partial_json_parser
7 from partial_json_parser.core.options import Allow
8 from transformers import PreTrainedTokenizerBase
9
10 -from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
10 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
11 + DeltaFunctionCall, DeltaMessage,
12 DeltaToolCall,
13 ExtractedToolCallInformation,
14 FunctionCall, ToolCall)
15 from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
15 - ToolParser)
16 + ToolParser, ToolParserManager)
17 from vllm.entrypoints.openai.tool_parsers.utils import find_common_prefix
18 from vllm.logger import init_logger
19 from vllm.utils import random_uuid
@@ -41,6 +42,7 @@ def is_complete_json(input_str):
42 return False
43
44
45 +@ToolParserManager.register_module("llama3_json")
46 class Llama3JsonToolParser(ToolParser):
47 """
48 Tool call parser for Llama 3.1 models intended for use with the
@@ -64,8 +66,9 @@ class Llama3JsonToolParser(ToolParser):
66 add_special_tokens=False)[0]
67 self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL)
68
67 - def extract_tool_calls(self,
68 - model_output: str) -> ExtractedToolCallInformation:
69 + def extract_tool_calls(
70 + self, model_output: str,
71 + request: ChatCompletionRequest) -> ExtractedToolCallInformation:
72 """
73 Extract the tool calls from a complete model response.
74 """
@@ -125,6 +128,7 @@ class Llama3JsonToolParser(ToolParser):
128 previous_token_ids: Sequence[int],
129 current_token_ids: Sequence[int],
130 delta_token_ids: Sequence[int],
131 + request: ChatCompletionRequest,
132 ) -> Union[DeltaMessage, None]:
133
134 if not (current_text.startswith(self.bot_token)
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+12 -8
@@ -8,12 +8,13 @@ import partial_json_parser
8 from partial_json_parser.core.options import Allow
9 from pydantic import Field
10
11 -from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
11 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12 + DeltaFunctionCall, DeltaMessage,
13 DeltaToolCall,
14 ExtractedToolCallInformation,
15 FunctionCall, ToolCall)
16 from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
16 - ToolParser)
17 + ToolParser, ToolParserManager)
18 from vllm.entrypoints.openai.tool_parsers.utils import (
19 extract_intermediate_diff)
20 from vllm.logger import init_logger
@@ -36,6 +37,7 @@ class MistralToolCall(ToolCall):
37 return "".join(choices(ALPHANUMERIC, k=9))
38
39
40 +@ToolParserManager.register_module("mistral")
41 class MistralToolParser(ToolParser):
42 """
43 Tool call parser for Mistral 7B Instruct v0.3, intended for use with the
@@ -47,9 +49,7 @@ class MistralToolParser(ToolParser):
49 def __init__(self, tokenizer: AnyTokenizer):
50 super().__init__(tokenizer)
51
50 - if isinstance(self.model_tokenizer, MistralTokenizer):
51 - self.model_tokenizer = self.model_tokenizer.tokenizer
52 - else:
52 + if not isinstance(self.model_tokenizer, MistralTokenizer):
53 logger.info("Non-Mistral tokenizer detected when using a Mistral "
54 "model...")
55
@@ -61,11 +61,14 @@ class MistralToolParser(ToolParser):
61 self.streamed_args_for_tool: List[str] = [
62 ] # map what has been streamed for each tool so far to a list
63 self.bot_token = "[TOOL_CALLS]"
64 - self.bot_token_id = self.model_tokenizer.vocab[self.bot_token]
64 + self.bot_token_id = self.model_tokenizer.get_vocab()[self.bot_token]
65 self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL)
66
67 - def extract_tool_calls(self,
68 - model_output: str) -> ExtractedToolCallInformation:
67 + def extract_tool_calls(
68 + self,
69 + model_output: str,
70 + request: ChatCompletionRequest,
71 + ) -> ExtractedToolCallInformation:
72 """
73 Extract the tool calls from a complete model response. Requires
74 find-and-replacing single quotes with double quotes for JSON parsing,
@@ -119,6 +122,7 @@ class MistralToolParser(ToolParser):
122 previous_token_ids: Sequence[int],
123 current_token_ids: Sequence[int],
124 delta_token_ids: Sequence[int],
125 + request: ChatCompletionRequest,
126 ) -> Union[DeltaMessage, None]:
127
128 # if the tool call token is not in the tokens generated so far, append