A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Model] Add mistral function calling format to all models loaded with "mistral" format (#8515)
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Patrick von Platen committed
Sep 17, 2024 at 19:50 UTC
a54ed8024953dc6b59906072a7a89cd4791ec4f0
5 files changed
+219
-9
examples/offline_chat_with_tools.py
new
+138
@@ -0,0 +1,138 @@
1
+# ruff: noqa
2
+import json
3
+import random
4
+import string
5
+
6
+from vllm import LLM
7
+from vllm.sampling_params import SamplingParams
8
+
9
+# This script is an offline demo for function calling
10
+#
11
+# If you want to run a server/client setup, please follow this code:
12
+#
13
+# - Server:
14
+#
15
+# ```bash
16
+# vllm serve mistralai/Mistral-7B-Instruct-v0.3 --tokenizer-mode mistral --load-format mistral --config-format mistral
17
+# ```
18
+#
19
+# - Client:
20
+#
21
+# ```bash
22
+# curl --location 'http://<your-node-url>:8000/v1/chat/completions' \
23
+# --header 'Content-Type: application/json' \
24
+# --header 'Authorization: Bearer token' \
25
+# --data '{
26
+# "model": "mistralai/Mistral-7B-Instruct-v0.3"
27
+# "messages": [
28
+# {
29
+# "role": "user",
30
+# "content": [
31
+# {"type" : "text", "text": "Describe this image in detail please."},
32
+# {"type": "image_url", "image_url": {"url": "https://s3.amazonaws.com/cms.ipressroom.com/338/files/201808/5b894ee1a138352221103195_A680%7Ejogging-edit/A680%7Ejogging-edit_hero.jpg"}},
33
+# {"type" : "text", "text": "and this one as well. Answer in French."},
34
+# {"type": "image_url", "image_url": {"url": "https://www.wolframcloud.com/obj/resourcesystem/images/a0e/a0ee3983-46c6-4c92-b85d-059044639928/6af8cfb971db031b.png"}}
35
+# ]
36
+# }
37
+# ]
38
+# }'
39
+# ```
40
+#
41
+# Usage:
42
+# python demo.py simple
43
+# python demo.py advanced
44
+
45
+model_name = "mistralai/Mistral-7B-Instruct-v0.3"
46
+# or switch to "mistralai/Mistral-Nemo-Instruct-2407"
47
+# or "mistralai/Mistral-Large-Instruct-2407"
48
+# or any other mistral model with function calling ability
49
+
50
+sampling_params = SamplingParams(max_tokens=8192, temperature=0.0)
51
+llm = LLM(model=model_name,
52
+ tokenizer_mode="mistral",
53
+ config_format="mistral",
54
+ load_format="mistral")
55
+
56
+
57
+def generate_random_id(length=9):
58
+ characters = string.ascii_letters + string.digits
59
+ random_id = ''.join(random.choice(characters) for _ in range(length))
60
+ return random_id
61
+
62
+
63
+# simulate an API that can be called
64
+def get_current_weather(city: str, state: str, unit: 'str'):
65
+ return (f"The weather in {city}, {state} is 85 degrees {unit}. It is "
66
+ "partly cloudly, with highs in the 90's.")
67
+
68
+
69
+tool_funtions = {"get_current_weather": get_current_weather}
70
+
71
+tools = [{
72
+ "type": "function",
73
+ "function": {
74
+ "name": "get_current_weather",
75
+ "description": "Get the current weather in a given location",
76
+ "parameters": {
77
+ "type": "object",
78
+ "properties": {
79
+ "city": {
80
+ "type":
81
+ "string",
82
+ "description":
83
+ "The city to find the weather for, e.g. 'San Francisco'"
84
+ },
85
+ "state": {
86
+ "type":
87
+ "string",
88
+ "description":
89
+ "the two-letter abbreviation for the state that the city is"
90
+ " in, e.g. 'CA' which would mean 'California'"
91
+ },
92
+ "unit": {
93
+ "type": "string",
94
+ "description": "The unit to fetch the temperature in",
95
+ "enum": ["celsius", "fahrenheit"]
96
+ }
97
+ },
98
+ "required": ["city", "state", "unit"]
99
+ }
100
+ }
101
+}]
102
+
103
+messages = [{
104
+ "role":
105
+ "user",
106
+ "content":
107
+ "Can you tell me what the temperate will be in Dallas, in fahrenheit?"
108
+}]
109
+
110
+outputs = llm.chat(messages, sampling_params=sampling_params, tools=tools)
111
+output = outputs[0].outputs[0].text.strip()
112
+
113
+# append the assistant message
114
+messages.append({
115
+ "role": "assistant",
116
+ "content": output,
117
+})
118
+
119
+# let's now actually parse and execute the model's output simulating an API call by using the
120
+# above defined function
121
+tool_calls = json.loads(output)
122
+tool_answers = [
123
+ tool_funtions[call['name']](**call['arguments']) for call in tool_calls
124
+]
125
+
126
+# append the answer as a tool message and let the LLM give you an answer
127
+messages.append({
128
+ "role": "tool",
129
+ "content": "\n\n".join(tool_answers),
130
+ "tool_call_id": generate_random_id(),
131
+})
132
+
133
+outputs = llm.chat(messages, sampling_params, tools=tools)
134
+
135
+print(outputs[0].outputs[0].text.strip())
136
+# yields
137
+# 'The weather in Dallas, TX is 85 degrees fahrenheit. '
138
+# 'It is partly cloudly, with highs in the 90's.'
tests/models/decoder_only/language/test_mistral.py
+67
@@ -4,13 +4,61 @@ Run `pytest tests/models/test_mistral.py`.
4
"""
5
import pytest
6
7
+from vllm import SamplingParams
8
+
9
from ...utils import check_logprobs_close
10
11
MODELS = [
12
"mistralai/Mistral-7B-Instruct-v0.1",
13
"mistralai/Mistral-7B-Instruct-v0.3",
14
+ # Mistral-Nemo is to big for CI, but passes locally
15
+ # "mistralai/Mistral-Nemo-Instruct-2407"
16
]
17
18
+SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5)
19
+
20
+# for function calling
21
+TOOLS = [{
22
+ "type": "function",
23
+ "function": {
24
+ "name": "get_current_weather",
25
+ "description": "Get the current weather in a given location",
26
+ "parameters": {
27
+ "type": "object",
28
+ "properties": {
29
+ "city": {
30
+ "type":
31
+ "string",
32
+ "description":
33
+ "The city to find the weather for, e.g. 'San Francisco'"
34
+ },
35
+ "state": {
36
+ "type":
37
+ "string",
38
+ "description":
39
+ "the two-letter abbreviation for the state that the city is"
40
+ " in, e.g. 'CA' which would mean 'California'"
41
+ },
42
+ "unit": {
43
+ "type": "string",
44
+ "description": "The unit to fetch the temperature in",
45
+ "enum": ["celsius", "fahrenheit"]
46
+ }
47
+ },
48
+ "required": ["city", "state", "unit"]
49
+ }
50
+ }
51
+}]
52
+MSGS = [{
53
+ "role":
54
+ "user",
55
+ "content": ("Can you tell me what the temperate"
56
+ " will be in Dallas, in fahrenheit?")
57
+}]
58
+EXPECTED_FUNC_CALL = (
59
+ '[{"name": "get_current_weather", "arguments": '
60
+ '{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]')
61
+
62
63
@pytest.mark.parametrize("model", MODELS)
64
@pytest.mark.parametrize("dtype", ["bfloat16"])
@@ -81,3 +129,22 @@ def test_mistral_format(
129
name_0="hf",
130
name_1="mistral",
131
)
132
+
133
+
134
+@pytest.mark.parametrize("dtype", ["bfloat16"])
135
+@pytest.mark.parametrize("model", MODELS[1:]) # v1 can't do func calling
136
+def test_mistral_function_calling(
137
+ vllm_runner,
138
+ model: str,
139
+ dtype: str,
140
+) -> None:
141
+ with vllm_runner(model,
142
+ dtype=dtype,
143
+ tokenizer_mode="mistral",
144
+ config_format="mistral",
145
+ load_format="mistral") as vllm_model:
146
+ outputs = vllm_model.model.chat(MSGS,
147
+ tools=TOOLS,
148
+ sampling_params=SAMPLING_PARAMS)
149
+
150
+ assert outputs[0].outputs[0].text.strip() == EXPECTED_FUNC_CALL
vllm/entrypoints/llm.py
+5
-1
@@ -1,5 +1,6 @@
1
from contextlib import contextmanager
2
-from typing import ClassVar, List, Optional, Sequence, Union, cast, overload
2
+from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Union, cast,
3
+ overload)
4
5
from tqdm import tqdm
6
@@ -357,6 +358,7 @@ class LLM:
358
lora_request: Optional[LoRARequest] = None,
359
chat_template: Optional[str] = None,
360
add_generation_prompt: bool = True,
361
+ tools: Optional[List[Dict[str, Any]]] = None,
362
) -> List[RequestOutput]:
363
"""
364
Generate responses for a chat conversation.
@@ -401,6 +403,7 @@ class LLM:
403
messages=messages,
404
chat_template=chat_template,
405
add_generation_prompt=add_generation_prompt,
406
+ tools=tools,
407
)
408
else:
409
prompt = apply_hf_chat_template(
@@ -408,6 +411,7 @@ class LLM:
411
conversation=conversation,
412
chat_template=chat_template,
413
add_generation_prompt=add_generation_prompt,
414
+ tools=tools,
415
)
416
417
inputs: PromptInputs
vllm/entrypoints/openai/serving_chat.py
+5
-4
@@ -123,7 +123,8 @@ class OpenAIServingChat(OpenAIServing):
123
]
124
125
prompt: Union[str, List[int]]
126
- if isinstance(tokenizer, MistralTokenizer):
126
+ is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer)
127
+ if is_mistral_tokenizer:
128
prompt = apply_mistral_chat_template(
129
tokenizer,
130
messages=request.messages,
@@ -159,10 +160,10 @@ class OpenAIServingChat(OpenAIServing):
160
return self.create_error_response(
161
"tool_choice = \"required\" is not supported!")
162
162
- # "auto" tools requires --enable-auto-tool-choice
163
- # and --tool-call-parser
164
- if request.tool_choice == "auto" and not (
163
+ if not is_mistral_tokenizer and request.tool_choice == "auto" and not (
164
self.enable_auto_tools and self.tool_parser is not None):
165
+ # for hf tokenizers, "auto" tools requires
166
+ # --enable-auto-tool-choice and --tool-call-parser
167
return self.create_error_response(
168
"\"auto\" tool choice requires "
169
"--enable-auto-tool-choice and --tool-call-parser to be set")
vllm/transformers_utils/tokenizers/mistral.py
+4
-4
@@ -165,10 +165,9 @@ class MistralTokenizer:
165
messages: List["ChatCompletionMessageParam"],
166
tools: Optional[Dict[str, Any]] = None,
167
**kwargs) -> List[int]:
168
- assert tools is None, "`tools` are not yet supported."
168
170
- request = ChatCompletionRequest(
171
- messages=messages) # type: ignore[type-var]
169
+ request = ChatCompletionRequest(messages=messages,
170
+ tools=tools) # type: ignore[type-var]
171
encoded = self.mistral.encode_chat_completion(request)
172
173
# encode-decode to get clean prompt
@@ -176,7 +175,8 @@ class MistralTokenizer:
175
176
def convert_tokens_to_string(self, tokens: List[str]) -> str:
177
if isinstance(self.tokenizer, Tekkenizer):
179
- return "".join(tokens)
178
+ return "".join(t for t in tokens
179
+ if t not in self.tokenizer._all_special_tokens)
180
else:
181
return self.tokenizer.decode(tokens) # type: ignore[arg-type]
182