A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Frontend] Add Phi-4-mini function calling support (#14886)
Signed-off-by: Kinfey <kinfeylo@microsoft.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Kinfey committed
Apr 1, 2025 at 13:50 UTC
a164aea35d48a4ef7316c203ef89720b0267d9b4
3 files changed
+170
-1
examples/tool_chat_template_phi4_mini.jinja
new
+60
@@ -0,0 +1,60 @@
1
+{%- if messages %}
2
+ {%- if system_message or tools %}
3
+<|system|>
4
+
5
+{%- if system_message %}
6
+{{ system_message }}
7
+{%- endif %}
8
+In addition to plain text responses, you can chose to call one or more of the provided functions.
9
+
10
+Use the following rule to decide when to call a function:
11
+ * if the response can be generated from your internal knowledge (e.g., as in the case of queries like "What is the capital of Poland?"), do so
12
+ * if you need external information that can be obtained by calling one or more of the provided functions, generate a function calls
13
+
14
+If you decide to call functions:
15
+ * prefix function calls with functools marker (no closing marker required)
16
+ * all function calls should be generated in a single JSON list formatted as functools[{"name": [function name], "arguments": [function arguments as JSON]}, ...]
17
+ * follow the provided JSON schema. Do not hallucinate arguments or values. Do to blindly copy values from the provided samples
18
+ * respect the argument type formatting. E.g., if the type if number and format is float, write value 7 as 7.0
19
+ * make sure you pick the right functions that match the user intent
20
+
21
+
22
+{%- if tools %}
23
+ {%- for t in tools %}
24
+ {{- t | tojson(indent=4) }}
25
+ {{- "\n\n" }}
26
+ {%- endfor %}
27
+{%- endif %}<|end|>
28
+ {%- endif %}
29
+
30
+ {%- for message in messages %}
31
+ {%- if message.role != "system" %}
32
+<|{{ message.role }}|>
33
+ {%- if message.content and message.role == "tools" %}
34
+{"result": {{ message.content }}}
35
+ {%- elif message.content %}
36
+{{ message.content }}
37
+ {%- elif message.tool_calls %}
38
+ {%- for call in message.tool_calls %}
39
+{"name": "{{ call.function.name }}", "arguments": {{ call.function.arguments }}}
40
+ {%- if not loop.last %},{% endif %}
41
+ {%- endfor %}
42
+ {%- endif %}<|end|>
43
+ {%- endif %}
44
+ {%- endfor %}<|assistant|>
45
+
46
+{%- else %}
47
+ {%- if system_message %}
48
+<|system|>
49
+
50
+{{ system_message }}<|end|>
51
+ {%- endif %}
52
+ {%- if prompt %}
53
+<|user|>
54
+
55
+{{ prompt }}<|end|>
56
+ {%- endif %}<|assistant|>
57
+
58
+{%- endif %}
59
+{{ response }}
60
+{%- if response %}<|user|>{% endif %}
\ No newline at end of file
vllm/entrypoints/openai/tool_parsers/__init__.py
+2
-1
@@ -8,11 +8,12 @@ from .internlm2_tool_parser import Internlm2ToolParser
8
from .jamba_tool_parser import JambaToolParser
9
from .llama_tool_parser import Llama3JsonToolParser
10
from .mistral_tool_parser import MistralToolParser
11
+from .phi4mini_tool_parser import Phi4MiniJsonToolParser
12
from .pythonic_tool_parser import PythonicToolParser
13
14
__all__ = [
15
"ToolParser", "ToolParserManager", "Granite20bFCToolParser",
16
"GraniteToolParser", "Hermes2ProToolParser", "MistralToolParser",
17
"Internlm2ToolParser", "Llama3JsonToolParser", "JambaToolParser",
17
- "PythonicToolParser"
18
+ "PythonicToolParser", "Phi4MiniJsonToolParser"
19
]
vllm/entrypoints/openai/tool_parsers/phi4mini_tool_parser.py
new
+108
@@ -0,0 +1,108 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+import json
4
+import re
5
+from collections.abc import Sequence
6
+from typing import Any, Optional
7
+
8
+from transformers import PreTrainedTokenizerBase
9
+
10
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
11
+ DeltaMessage,
12
+ ExtractedToolCallInformation,
13
+ FunctionCall, ToolCall)
14
+from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
15
+ ToolParser, ToolParserManager)
16
+from vllm.logger import init_logger
17
+from vllm.utils import random_uuid
18
+
19
+logger = init_logger(__name__)
20
+
21
+
22
+@ToolParserManager.register_module("phi4_mini_json")
23
+class Phi4MiniJsonToolParser(ToolParser):
24
+ """
25
+ Tool call parser for phi-4-mini models intended for use with the
26
+ examples/tool_chat_template_llama.jinja template.
27
+
28
+ Used when --enable-auto-tool-choice --tool-call-parser phi4_mini_json
29
+ are all set
30
+ """
31
+
32
+ def __init__(self, tokenizer: PreTrainedTokenizerBase) -> None:
33
+ super().__init__(tokenizer)
34
+
35
+ # initialize properties used for state when parsing tool calls in
36
+ # streaming mode
37
+ self.prev_tool_call_arr: list[dict[str, Any]] = []
38
+ self.current_tool_id: int = -1
39
+ self.current_tool_name_sent: bool = False
40
+ self.streamed_args_for_tool: list[str] = [
41
+ ] # map what has been streamed for each tool so far to a list
42
+ self.bot_token: str = "functools"
43
+
44
+ def extract_tool_calls(
45
+ self, model_output: str,
46
+ request: ChatCompletionRequest) -> ExtractedToolCallInformation:
47
+ """
48
+ Extract the tool calls from a complete model response.
49
+ """
50
+ print(f"Model output: {model_output}")
51
+
52
+ pattern = r'functools\[(.*?)\]'
53
+ matches = re.search(pattern, model_output, re.DOTALL)
54
+
55
+ if not matches:
56
+ print("No function calls found")
57
+ return ExtractedToolCallInformation(tools_called=False,
58
+ tool_calls=[],
59
+ content=model_output)
60
+
61
+ try:
62
+ function_call_arr: list[dict[str, Any]] = []
63
+ try:
64
+ json_content = '[' + matches.group(1) + ']'
65
+
66
+ function_call_arr = json.loads(json_content)
67
+ print(f"Successfully extracted {len(function_call_arr)} "
68
+ "function calls")
69
+ except json.JSONDecodeError as e:
70
+ print(f"Error parsing JSON: {e}")
71
+
72
+ tool_calls: list[ToolCall] = [
73
+ ToolCall(
74
+ id=f"chatcmpl-tool-{random_uuid()}",
75
+ type="function",
76
+ function=FunctionCall(
77
+ name=raw_function_call["name"],
78
+ # function call args are JSON but as a string
79
+ arguments=json.dumps(
80
+ raw_function_call["arguments"] if "arguments" in
81
+ raw_function_call else
82
+ raw_function_call["parameters"])))
83
+ for raw_function_call in function_call_arr
84
+ ]
85
+
86
+ # get any content before the tool call
87
+ ret = ExtractedToolCallInformation(tools_called=True,
88
+ tool_calls=tool_calls,
89
+ content=None)
90
+ return ret
91
+
92
+ except Exception:
93
+ return ExtractedToolCallInformation(tools_called=False,
94
+ tool_calls=[],
95
+ content=model_output)
96
+
97
+ def extract_tool_calls_streaming(
98
+ self,
99
+ previous_text: str,
100
+ current_text: str,
101
+ delta_text: str,
102
+ previous_token_ids: Sequence[int],
103
+ current_token_ids: Sequence[int],
104
+ delta_token_ids: Sequence[int],
105
+ request: ChatCompletionRequest,
106
+ ) -> Optional[DeltaMessage]:
107
+
108
+ return None