A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[FRONTEND] OpenAI `tools` support named functions (#5032)
Breno Faria committed
Jun 4, 2024 at 01:25 UTC
f775a07e30fdeafc14f53fe502b262b00540dd71
6 files changed
+314
-11
docs/source/serving/openai_compatible_server.md
+12
-1
@@ -109,4 +109,15 @@ directory [here](https://github.com/vllm-project/vllm/tree/main/examples/)
109
:module: vllm.entrypoints.openai.cli_args
110
:func: make_arg_parser
111
:prog: -m vllm.entrypoints.openai.api_server
112
-```
\ No newline at end of file
112
+```
113
+
114
+## Tool calling in the chat completion API
115
+vLLM supports only named function calling in the chat completion API. The `tool_choice` options `auto` and `required` are **not yet supported** but on the roadmap.
116
+
117
+To use a named function you need to define the function in the `tools` parameter and call it in the `tool_choice` parameter.
118
+
119
+It is the callers responsibility to prompt the model with the tool information, vLLM will not automatically manipulate the prompt. **This may change in the future.**
120
+
121
+vLLM will use guided decoding to ensure the response matches the tool parameter object defined by the JSON schema in the `tools` parameter.
122
+
123
+Please refer to the OpenAI API reference documentation for more information.
\ No newline at end of file
tests/entrypoints/test_openai_server.py
+185
@@ -906,6 +906,191 @@ async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
906
for token in top_logprobs)
907
908
909
+@pytest.mark.asyncio
910
+@pytest.mark.parametrize("guided_decoding_backend",
911
+ ["outlines", "lm-format-enforcer"])
912
+async def test_named_tool_use(server, client: openai.AsyncOpenAI,
913
+ guided_decoding_backend: str):
914
+ messages = [{
915
+ "role": "system",
916
+ "content": "you are a helpful assistant"
917
+ }, {
918
+ "role":
919
+ "user",
920
+ "content":
921
+ f"Give an example JSON for an employee profile that "
922
+ f"fits this schema: {TEST_SCHEMA}"
923
+ }]
924
+
925
+ # non-streaming
926
+
927
+ chat_completion = await client.chat.completions.create(
928
+ model=MODEL_NAME,
929
+ messages=messages,
930
+ max_tokens=1000,
931
+ tools=[{
932
+ "type": "function",
933
+ "function": {
934
+ "name": "dummy_function_name",
935
+ "description": "This is a dummy function",
936
+ "parameters": TEST_SCHEMA
937
+ }
938
+ }],
939
+ tool_choice={
940
+ "type": "function",
941
+ "function": {
942
+ "name": "dummy_function_name"
943
+ }
944
+ })
945
+ message = chat_completion.choices[0].message
946
+ assert len(message.content) == 0
947
+ json_string = message.tool_calls[0].function.arguments
948
+ json1 = json.loads(json_string)
949
+ jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
950
+
951
+ messages.append({"role": "assistant", "content": json_string})
952
+ messages.append({
953
+ "role":
954
+ "user",
955
+ "content":
956
+ "Give me another one with a different name and age"
957
+ })
958
+
959
+ # streaming
960
+
961
+ stream = await client.chat.completions.create(
962
+ model=MODEL_NAME,
963
+ messages=messages,
964
+ max_tokens=1000,
965
+ tools=[{
966
+ "type": "function",
967
+ "function": {
968
+ "name": "dummy_function_name",
969
+ "description": "This is a dummy function",
970
+ "parameters": TEST_SCHEMA
971
+ }
972
+ }],
973
+ tool_choice={
974
+ "type": "function",
975
+ "function": {
976
+ "name": "dummy_function_name"
977
+ }
978
+ },
979
+ stream=True)
980
+
981
+ output = []
982
+ finish_reason_count = 0
983
+ async for chunk in stream:
984
+ delta = chunk.choices[0].delta
985
+ if delta.role:
986
+ assert delta.role == "assistant"
987
+ assert delta.content is None or len(delta.content) == 0
988
+ if delta.tool_calls:
989
+ output.append(delta.tool_calls[0].function.arguments)
990
+ if chunk.choices[0].finish_reason is not None:
991
+ finish_reason_count += 1
992
+ # finish reason should only return in last block
993
+ assert finish_reason_count == 1
994
+ json2 = json.loads("".join(output))
995
+ jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
996
+ assert json1["name"] != json2["name"]
997
+ assert json1["age"] != json2["age"]
998
+
999
+
1000
+@pytest.mark.asyncio
1001
+@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
1002
+async def test_required_tool_use_not_yet_supported(
1003
+ server, client: openai.AsyncOpenAI, guided_decoding_backend: str):
1004
+ messages = [{
1005
+ "role": "system",
1006
+ "content": "you are a helpful assistant"
1007
+ }, {
1008
+ "role":
1009
+ "user",
1010
+ "content":
1011
+ f"Give an example JSON for an employee profile that "
1012
+ f"fits this schema: {TEST_SCHEMA}"
1013
+ }]
1014
+
1015
+ with pytest.raises(openai.BadRequestError):
1016
+ await client.chat.completions.create(
1017
+ model=MODEL_NAME,
1018
+ messages=messages,
1019
+ max_tokens=1000,
1020
+ tools=[{
1021
+ "type": "function",
1022
+ "function": {
1023
+ "name": "dummy_function_name",
1024
+ "description": "This is a dummy function",
1025
+ "parameters": TEST_SCHEMA
1026
+ }
1027
+ }],
1028
+ tool_choice="required")
1029
+
1030
+ with pytest.raises(openai.BadRequestError):
1031
+ await client.chat.completions.create(
1032
+ model=MODEL_NAME,
1033
+ messages=messages,
1034
+ max_tokens=1000,
1035
+ tools=[{
1036
+ "type": "function",
1037
+ "function": {
1038
+ "name": "dummy_function_name",
1039
+ "description": "This is a dummy function",
1040
+ "parameters": TEST_SCHEMA
1041
+ }
1042
+ }],
1043
+ tool_choice="auto")
1044
+
1045
+
1046
+@pytest.mark.asyncio
1047
+@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
1048
+async def test_inconsistent_tool_choice_and_tools(
1049
+ server, client: openai.AsyncOpenAI, guided_decoding_backend: str):
1050
+ messages = [{
1051
+ "role": "system",
1052
+ "content": "you are a helpful assistant"
1053
+ }, {
1054
+ "role":
1055
+ "user",
1056
+ "content":
1057
+ f"Give an example JSON for an employee profile that "
1058
+ f"fits this schema: {TEST_SCHEMA}"
1059
+ }]
1060
+
1061
+ with pytest.raises(openai.BadRequestError):
1062
+ await client.chat.completions.create(model=MODEL_NAME,
1063
+ messages=messages,
1064
+ max_tokens=1000,
1065
+ tool_choice={
1066
+ "type": "function",
1067
+ "function": {
1068
+ "name":
1069
+ "dummy_function_name"
1070
+ }
1071
+ })
1072
+
1073
+ with pytest.raises(openai.BadRequestError):
1074
+ await client.chat.completions.create(
1075
+ model=MODEL_NAME,
1076
+ messages=messages,
1077
+ max_tokens=1000,
1078
+ tools=[{
1079
+ "type": "function",
1080
+ "function": {
1081
+ "name": "dummy_function_name",
1082
+ "description": "This is a dummy function",
1083
+ "parameters": TEST_SCHEMA
1084
+ }
1085
+ }],
1086
+ tool_choice={
1087
+ "type": "function",
1088
+ "function": {
1089
+ "name": "nondefined_function_name"
1090
+ }
1091
+ })
1092
+
1093
+
1094
@pytest.mark.asyncio
1095
async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
1096
for _ in range(2):
tests/utils.py
+2
-1
@@ -24,7 +24,8 @@ class ServerRunner:
24
env = os.environ.copy()
25
env["PYTHONUNBUFFERED"] = "1"
26
self.proc = subprocess.Popen(
27
- ["python3", "-m", "vllm.entrypoints.openai.api_server"] + args,
27
+ [sys.executable, "-m", "vllm.entrypoints.openai.api_server"] +
28
+ args,
29
env=env,
30
stdout=sys.stdout,
31
stderr=sys.stderr,
vllm/entrypoints/openai/protocol.py
+55
-2
@@ -102,6 +102,26 @@ class ResponseFormat(OpenAIBaseModel):
102
type: Literal["text", "json_object"]
103
104
105
+class FunctionDefinition(OpenAIBaseModel):
106
+ name: str
107
+ description: Optional[str] = None
108
+ parameters: Optional[Dict[str, Any]] = None
109
+
110
+
111
+class ChatCompletionToolsParam(OpenAIBaseModel):
112
+ type: Literal["function"] = "function"
113
+ function: FunctionDefinition
114
+
115
+
116
+class ChatCompletionNamedFunction(OpenAIBaseModel):
117
+ name: str
118
+
119
+
120
+class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
121
+ function: ChatCompletionNamedFunction
122
+ type: Literal["function"] = "function"
123
+
124
+
125
class ChatCompletionRequest(OpenAIBaseModel):
126
# Ordered by official OpenAI API documentation
127
# https://platform.openai.com/docs/api-reference/chat/create
@@ -122,6 +142,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
142
stream: Optional[bool] = False
143
temperature: Optional[float] = 0.7
144
top_p: Optional[float] = 1.0
145
+ tools: Optional[List[ChatCompletionToolsParam]] = None
146
+ tool_choice: Optional[Union[Literal["none"],
147
+ ChatCompletionNamedToolChoiceParam]] = "none"
148
user: Optional[str] = None
149
150
# doc: begin-chat-completion-sampling-params
@@ -245,10 +268,27 @@ class ChatCompletionRequest(OpenAIBaseModel):
268
"guided_regex" in data and data["guided_regex"] is not None,
269
"guided_choice" in data and data["guided_choice"] is not None
270
])
271
+ # you can only use one kind of guided decoding
272
if guide_count > 1:
273
raise ValueError(
274
"You can only use one kind of guided decoding "
275
"('guided_json', 'guided_regex' or 'guided_choice').")
276
+ # you can only either use guided decoding or tools, not both
277
+ if guide_count > 1 and "tool_choice" in data and data[
278
+ "tool_choice"] != "none":
279
+ raise ValueError(
280
+ "You can only either use guided decoding or tools, not both.")
281
+ return data
282
+
283
+ @model_validator(mode="before")
284
+ @classmethod
285
+ def check_tool_choice(cls, data):
286
+ if "tool_choice" in data and data["tool_choice"] != "none":
287
+ if not isinstance(data["tool_choice"], dict):
288
+ raise ValueError("Currently only named tools are supported.")
289
+ if "tools" not in data or data["tools"] is None:
290
+ raise ValueError(
291
+ "When using `tool_choice`, `tools` must be set.")
292
return data
293
294
@model_validator(mode="before")
@@ -506,9 +546,21 @@ class EmbeddingResponse(BaseModel):
546
usage: UsageInfo
547
548
549
+class FunctionCall(OpenAIBaseModel):
550
+ name: str
551
+ arguments: str
552
+
553
+
554
+class ToolCall(OpenAIBaseModel):
555
+ id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
556
+ type: Literal["function"] = "function"
557
+ function: FunctionCall
558
+
559
+
560
class ChatMessage(OpenAIBaseModel):
561
role: str
562
content: str
563
+ tool_calls: List[ToolCall] = Field(default_factory=list)
564
565
566
class ChatCompletionLogProb(OpenAIBaseModel):
@@ -535,7 +587,7 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
587
588
class ChatCompletionResponse(OpenAIBaseModel):
589
id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
538
- object: str = "chat.completion"
590
+ object: Literal["chat.completion"] = "chat.completion"
591
created: int = Field(default_factory=lambda: int(time.time()))
592
model: str
593
choices: List[ChatCompletionResponseChoice]
@@ -545,6 +597,7 @@ class ChatCompletionResponse(OpenAIBaseModel):
597
class DeltaMessage(OpenAIBaseModel):
598
role: Optional[str] = None
599
content: Optional[str] = None
600
+ tool_calls: List[ToolCall] = Field(default_factory=list)
601
602
603
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
@@ -557,7 +610,7 @@ class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
610
611
class ChatCompletionStreamResponse(OpenAIBaseModel):
612
id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
560
- object: str = "chat.completion.chunk"
613
+ object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
614
created: int = Field(default_factory=lambda: int(time.time()))
615
model: str
616
choices: List[ChatCompletionResponseStreamChoice]
vllm/entrypoints/openai/serving_chat.py
+32
-5
@@ -14,10 +14,11 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
14
from vllm.entrypoints.openai.protocol import (
15
ChatCompletionContentPartParam, ChatCompletionLogProb,
16
ChatCompletionLogProbs, ChatCompletionLogProbsContent,
17
- ChatCompletionMessageParam, ChatCompletionRequest, ChatCompletionResponse,
17
+ ChatCompletionMessageParam, ChatCompletionNamedToolChoiceParam,
18
+ ChatCompletionRequest, ChatCompletionResponse,
19
ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
20
ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
20
- UsageInfo)
21
+ FunctionCall, ToolCall, UsageInfo)
22
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
23
OpenAIServing)
24
from vllm.logger import init_logger
@@ -298,11 +299,24 @@ class OpenAIServingChat(OpenAIServing):
299
delta_text = output.text[len(previous_texts[i]):]
300
previous_texts[i] = output.text
301
previous_num_tokens[i] = len(output.token_ids)
302
+
303
+ if request.tool_choice and type(
304
+ request.tool_choice
305
+ ) is ChatCompletionNamedToolChoiceParam:
306
+ delta_message = DeltaMessage(tool_calls=[
307
+ ToolCall(function=FunctionCall(
308
+ name=request.tool_choice.function.name,
309
+ arguments=delta_text))
310
+ ])
311
+ else:
312
+ delta_message = DeltaMessage(content=delta_text)
313
+
314
if output.finish_reason is None:
315
# Send token-by-token response for each request.n
316
+
317
choice_data = ChatCompletionResponseStreamChoice(
318
index=i,
305
- delta=DeltaMessage(content=delta_text),
319
+ delta=delta_message,
320
logprobs=logprobs,
321
finish_reason=None)
322
chunk = ChatCompletionStreamResponse(
@@ -324,7 +338,7 @@ class OpenAIServingChat(OpenAIServing):
338
)
339
choice_data = ChatCompletionResponseStreamChoice(
340
index=i,
327
- delta=DeltaMessage(content=delta_text),
341
+ delta=delta_message,
342
logprobs=logprobs,
343
finish_reason=output.finish_reason,
344
stop_reason=output.stop_reason)
@@ -381,9 +395,22 @@ class OpenAIServingChat(OpenAIServing):
395
else:
396
logprobs = None
397
398
+ if request.tool_choice and type(
399
+ request.tool_choice) is ChatCompletionNamedToolChoiceParam:
400
+ message = ChatMessage(
401
+ role=role,
402
+ content="",
403
+ tool_calls=[
404
+ ToolCall(function=FunctionCall(
405
+ name=request.tool_choice.function.name,
406
+ arguments=output.text))
407
+ ])
408
+ elif not request.tool_choice or request.tool_choice == "none":
409
+ message = ChatMessage(role=role, content=output.text)
410
+
411
choice_data = ChatCompletionResponseChoice(
412
index=output.index,
386
- message=ChatMessage(role=role, content=output.text),
413
+ message=message,
414
logprobs=logprobs,
415
finish_reason=output.finish_reason,
416
stop_reason=output.stop_reason)
vllm/model_executor/guided_decoding/__init__.py
+28
-2
@@ -1,7 +1,8 @@
1
from typing import Optional, Union
2
3
-from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
4
- CompletionRequest)
3
+from vllm.entrypoints.openai.protocol import (
4
+ ChatCompletionNamedToolChoiceParam, ChatCompletionRequest,
5
+ CompletionRequest)
6
from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import (
7
get_lm_format_enforcer_guided_decoding_logits_processor)
8
from vllm.model_executor.guided_decoding.outlines_decoding import (
@@ -13,6 +14,8 @@ async def get_guided_decoding_logits_processor(
14
guided_decoding_backend: str, request: Union[CompletionRequest,
15
ChatCompletionRequest],
16
tokenizer) -> Optional[LogitsProcessor]:
17
+ request = _adapt_request_for_tool_use(request)
18
+
19
if guided_decoding_backend == 'outlines':
20
return await get_outlines_guided_decoding_logits_processor(
21
request, tokenizer)
@@ -23,3 +26,26 @@ async def get_guided_decoding_logits_processor(
26
raise ValueError(
27
f"Unknown guided decoding backend '{guided_decoding_backend}'. "
28
"Must be one of 'outlines, 'lm-format-enforcer'")
29
+
30
+
31
+def _adapt_request_for_tool_use(request: Union[CompletionRequest,
32
+ ChatCompletionRequest]):
33
+ # the legacy completion API does not support tool use
34
+ if type(request) is CompletionRequest:
35
+ return request
36
+
37
+ # user has chosen to not use any tool
38
+ if request.tool_choice == "none":
39
+ return request
40
+
41
+ # user has chosen to use a named tool
42
+ if type(request.tool_choice) is ChatCompletionNamedToolChoiceParam:
43
+ tool_name = request.tool_choice.function.name
44
+ tools = {tool.function.name: tool.function for tool in request.tools}
45
+ if tool_name not in tools:
46
+ raise ValueError(
47
+ f"Tool '{tool_name}' has not been passed in `tools`.")
48
+ tool = tools[tool_name]
49
+ request.guided_json = tool.parameters
50
+
51
+ return request