@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f775a07e30
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[FRONTEND] OpenAI `tools` support named functions (#5032)

Breno Faria committed Jun 4, 2024 at 01:25 UTC f775a07e30fdeafc14f53fe502b262b00540dd71
6 files changed +314 -11
docs/source/serving/openai_compatible_server.md
+12 -1
@@ -109,4 +109,15 @@ directory [here](https://github.com/vllm-project/vllm/tree/main/examples/)
109 :module: vllm.entrypoints.openai.cli_args
110 :func: make_arg_parser
111 :prog: -m vllm.entrypoints.openai.api_server
112 -```
\ No newline at end of file
112 +```
113 +
114 +## Tool calling in the chat completion API
115 +vLLM supports only named function calling in the chat completion API. The `tool_choice` options `auto` and `required` are **not yet supported** but on the roadmap.
116 +
117 +To use a named function you need to define the function in the `tools` parameter and call it in the `tool_choice` parameter.
118 +
119 +It is the callers responsibility to prompt the model with the tool information, vLLM will not automatically manipulate the prompt. **This may change in the future.**
120 +
121 +vLLM will use guided decoding to ensure the response matches the tool parameter object defined by the JSON schema in the `tools` parameter.
122 +
123 +Please refer to the OpenAI API reference documentation for more information.
\ No newline at end of file
tests/entrypoints/test_openai_server.py
+185
@@ -906,6 +906,191 @@ async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
906 for token in top_logprobs)
907
908
909 +@pytest.mark.asyncio
910 +@pytest.mark.parametrize("guided_decoding_backend",
911 + ["outlines", "lm-format-enforcer"])
912 +async def test_named_tool_use(server, client: openai.AsyncOpenAI,
913 + guided_decoding_backend: str):
914 + messages = [{
915 + "role": "system",
916 + "content": "you are a helpful assistant"
917 + }, {
918 + "role":
919 + "user",
920 + "content":
921 + f"Give an example JSON for an employee profile that "
922 + f"fits this schema: {TEST_SCHEMA}"
923 + }]
924 +
925 + # non-streaming
926 +
927 + chat_completion = await client.chat.completions.create(
928 + model=MODEL_NAME,
929 + messages=messages,
930 + max_tokens=1000,
931 + tools=[{
932 + "type": "function",
933 + "function": {
934 + "name": "dummy_function_name",
935 + "description": "This is a dummy function",
936 + "parameters": TEST_SCHEMA
937 + }
938 + }],
939 + tool_choice={
940 + "type": "function",
941 + "function": {
942 + "name": "dummy_function_name"
943 + }
944 + })
945 + message = chat_completion.choices[0].message
946 + assert len(message.content) == 0
947 + json_string = message.tool_calls[0].function.arguments
948 + json1 = json.loads(json_string)
949 + jsonschema.validate(instance=json1, schema=TEST_SCHEMA)
950 +
951 + messages.append({"role": "assistant", "content": json_string})
952 + messages.append({
953 + "role":
954 + "user",
955 + "content":
956 + "Give me another one with a different name and age"
957 + })
958 +
959 + # streaming
960 +
961 + stream = await client.chat.completions.create(
962 + model=MODEL_NAME,
963 + messages=messages,
964 + max_tokens=1000,
965 + tools=[{
966 + "type": "function",
967 + "function": {
968 + "name": "dummy_function_name",
969 + "description": "This is a dummy function",
970 + "parameters": TEST_SCHEMA
971 + }
972 + }],
973 + tool_choice={
974 + "type": "function",
975 + "function": {
976 + "name": "dummy_function_name"
977 + }
978 + },
979 + stream=True)
980 +
981 + output = []
982 + finish_reason_count = 0
983 + async for chunk in stream:
984 + delta = chunk.choices[0].delta
985 + if delta.role:
986 + assert delta.role == "assistant"
987 + assert delta.content is None or len(delta.content) == 0
988 + if delta.tool_calls:
989 + output.append(delta.tool_calls[0].function.arguments)
990 + if chunk.choices[0].finish_reason is not None:
991 + finish_reason_count += 1
992 + # finish reason should only return in last block
993 + assert finish_reason_count == 1
994 + json2 = json.loads("".join(output))
995 + jsonschema.validate(instance=json2, schema=TEST_SCHEMA)
996 + assert json1["name"] != json2["name"]
997 + assert json1["age"] != json2["age"]
998 +
999 +
1000 +@pytest.mark.asyncio
1001 +@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
1002 +async def test_required_tool_use_not_yet_supported(
1003 + server, client: openai.AsyncOpenAI, guided_decoding_backend: str):
1004 + messages = [{
1005 + "role": "system",
1006 + "content": "you are a helpful assistant"
1007 + }, {
1008 + "role":
1009 + "user",
1010 + "content":
1011 + f"Give an example JSON for an employee profile that "
1012 + f"fits this schema: {TEST_SCHEMA}"
1013 + }]
1014 +
1015 + with pytest.raises(openai.BadRequestError):
1016 + await client.chat.completions.create(
1017 + model=MODEL_NAME,
1018 + messages=messages,
1019 + max_tokens=1000,
1020 + tools=[{
1021 + "type": "function",
1022 + "function": {
1023 + "name": "dummy_function_name",
1024 + "description": "This is a dummy function",
1025 + "parameters": TEST_SCHEMA
1026 + }
1027 + }],
1028 + tool_choice="required")
1029 +
1030 + with pytest.raises(openai.BadRequestError):
1031 + await client.chat.completions.create(
1032 + model=MODEL_NAME,
1033 + messages=messages,
1034 + max_tokens=1000,
1035 + tools=[{
1036 + "type": "function",
1037 + "function": {
1038 + "name": "dummy_function_name",
1039 + "description": "This is a dummy function",
1040 + "parameters": TEST_SCHEMA
1041 + }
1042 + }],
1043 + tool_choice="auto")
1044 +
1045 +
1046 +@pytest.mark.asyncio
1047 +@pytest.mark.parametrize("guided_decoding_backend", ["outlines"])
1048 +async def test_inconsistent_tool_choice_and_tools(
1049 + server, client: openai.AsyncOpenAI, guided_decoding_backend: str):
1050 + messages = [{
1051 + "role": "system",
1052 + "content": "you are a helpful assistant"
1053 + }, {
1054 + "role":
1055 + "user",
1056 + "content":
1057 + f"Give an example JSON for an employee profile that "
1058 + f"fits this schema: {TEST_SCHEMA}"
1059 + }]
1060 +
1061 + with pytest.raises(openai.BadRequestError):
1062 + await client.chat.completions.create(model=MODEL_NAME,
1063 + messages=messages,
1064 + max_tokens=1000,
1065 + tool_choice={
1066 + "type": "function",
1067 + "function": {
1068 + "name":
1069 + "dummy_function_name"
1070 + }
1071 + })
1072 +
1073 + with pytest.raises(openai.BadRequestError):
1074 + await client.chat.completions.create(
1075 + model=MODEL_NAME,
1076 + messages=messages,
1077 + max_tokens=1000,
1078 + tools=[{
1079 + "type": "function",
1080 + "function": {
1081 + "name": "dummy_function_name",
1082 + "description": "This is a dummy function",
1083 + "parameters": TEST_SCHEMA
1084 + }
1085 + }],
1086 + tool_choice={
1087 + "type": "function",
1088 + "function": {
1089 + "name": "nondefined_function_name"
1090 + }
1091 + })
1092 +
1093 +
1094 @pytest.mark.asyncio
1095 async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
1096 for _ in range(2):
tests/utils.py
+2 -1
@@ -24,7 +24,8 @@ class ServerRunner:
24 env = os.environ.copy()
25 env["PYTHONUNBUFFERED"] = "1"
26 self.proc = subprocess.Popen(
27 - ["python3", "-m", "vllm.entrypoints.openai.api_server"] + args,
27 + [sys.executable, "-m", "vllm.entrypoints.openai.api_server"] +
28 + args,
29 env=env,
30 stdout=sys.stdout,
31 stderr=sys.stderr,
vllm/entrypoints/openai/protocol.py
+55 -2
@@ -102,6 +102,26 @@ class ResponseFormat(OpenAIBaseModel):
102 type: Literal["text", "json_object"]
103
104
105 +class FunctionDefinition(OpenAIBaseModel):
106 + name: str
107 + description: Optional[str] = None
108 + parameters: Optional[Dict[str, Any]] = None
109 +
110 +
111 +class ChatCompletionToolsParam(OpenAIBaseModel):
112 + type: Literal["function"] = "function"
113 + function: FunctionDefinition
114 +
115 +
116 +class ChatCompletionNamedFunction(OpenAIBaseModel):
117 + name: str
118 +
119 +
120 +class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
121 + function: ChatCompletionNamedFunction
122 + type: Literal["function"] = "function"
123 +
124 +
125 class ChatCompletionRequest(OpenAIBaseModel):
126 # Ordered by official OpenAI API documentation
127 # https://platform.openai.com/docs/api-reference/chat/create
@@ -122,6 +142,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
142 stream: Optional[bool] = False
143 temperature: Optional[float] = 0.7
144 top_p: Optional[float] = 1.0
145 + tools: Optional[List[ChatCompletionToolsParam]] = None
146 + tool_choice: Optional[Union[Literal["none"],
147 + ChatCompletionNamedToolChoiceParam]] = "none"
148 user: Optional[str] = None
149
150 # doc: begin-chat-completion-sampling-params
@@ -245,10 +268,27 @@ class ChatCompletionRequest(OpenAIBaseModel):
268 "guided_regex" in data and data["guided_regex"] is not None,
269 "guided_choice" in data and data["guided_choice"] is not None
270 ])
271 + # you can only use one kind of guided decoding
272 if guide_count > 1:
273 raise ValueError(
274 "You can only use one kind of guided decoding "
275 "('guided_json', 'guided_regex' or 'guided_choice').")
276 + # you can only either use guided decoding or tools, not both
277 + if guide_count > 1 and "tool_choice" in data and data[
278 + "tool_choice"] != "none":
279 + raise ValueError(
280 + "You can only either use guided decoding or tools, not both.")
281 + return data
282 +
283 + @model_validator(mode="before")
284 + @classmethod
285 + def check_tool_choice(cls, data):
286 + if "tool_choice" in data and data["tool_choice"] != "none":
287 + if not isinstance(data["tool_choice"], dict):
288 + raise ValueError("Currently only named tools are supported.")
289 + if "tools" not in data or data["tools"] is None:
290 + raise ValueError(
291 + "When using `tool_choice`, `tools` must be set.")
292 return data
293
294 @model_validator(mode="before")
@@ -506,9 +546,21 @@ class EmbeddingResponse(BaseModel):
546 usage: UsageInfo
547
548
549 +class FunctionCall(OpenAIBaseModel):
550 + name: str
551 + arguments: str
552 +
553 +
554 +class ToolCall(OpenAIBaseModel):
555 + id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
556 + type: Literal["function"] = "function"
557 + function: FunctionCall
558 +
559 +
560 class ChatMessage(OpenAIBaseModel):
561 role: str
562 content: str
563 + tool_calls: List[ToolCall] = Field(default_factory=list)
564
565
566 class ChatCompletionLogProb(OpenAIBaseModel):
@@ -535,7 +587,7 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
587
588 class ChatCompletionResponse(OpenAIBaseModel):
589 id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
538 - object: str = "chat.completion"
590 + object: Literal["chat.completion"] = "chat.completion"
591 created: int = Field(default_factory=lambda: int(time.time()))
592 model: str
593 choices: List[ChatCompletionResponseChoice]
@@ -545,6 +597,7 @@ class ChatCompletionResponse(OpenAIBaseModel):
597 class DeltaMessage(OpenAIBaseModel):
598 role: Optional[str] = None
599 content: Optional[str] = None
600 + tool_calls: List[ToolCall] = Field(default_factory=list)
601
602
603 class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
@@ -557,7 +610,7 @@ class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
610
611 class ChatCompletionStreamResponse(OpenAIBaseModel):
612 id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
560 - object: str = "chat.completion.chunk"
613 + object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
614 created: int = Field(default_factory=lambda: int(time.time()))
615 model: str
616 choices: List[ChatCompletionResponseStreamChoice]
vllm/entrypoints/openai/serving_chat.py
+32 -5
@@ -14,10 +14,11 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
14 from vllm.entrypoints.openai.protocol import (
15 ChatCompletionContentPartParam, ChatCompletionLogProb,
16 ChatCompletionLogProbs, ChatCompletionLogProbsContent,
17 - ChatCompletionMessageParam, ChatCompletionRequest, ChatCompletionResponse,
17 + ChatCompletionMessageParam, ChatCompletionNamedToolChoiceParam,
18 + ChatCompletionRequest, ChatCompletionResponse,
19 ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
20 ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
20 - UsageInfo)
21 + FunctionCall, ToolCall, UsageInfo)
22 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
23 OpenAIServing)
24 from vllm.logger import init_logger
@@ -298,11 +299,24 @@ class OpenAIServingChat(OpenAIServing):
299 delta_text = output.text[len(previous_texts[i]):]
300 previous_texts[i] = output.text
301 previous_num_tokens[i] = len(output.token_ids)
302 +
303 + if request.tool_choice and type(
304 + request.tool_choice
305 + ) is ChatCompletionNamedToolChoiceParam:
306 + delta_message = DeltaMessage(tool_calls=[
307 + ToolCall(function=FunctionCall(
308 + name=request.tool_choice.function.name,
309 + arguments=delta_text))
310 + ])
311 + else:
312 + delta_message = DeltaMessage(content=delta_text)
313 +
314 if output.finish_reason is None:
315 # Send token-by-token response for each request.n
316 +
317 choice_data = ChatCompletionResponseStreamChoice(
318 index=i,
305 - delta=DeltaMessage(content=delta_text),
319 + delta=delta_message,
320 logprobs=logprobs,
321 finish_reason=None)
322 chunk = ChatCompletionStreamResponse(
@@ -324,7 +338,7 @@ class OpenAIServingChat(OpenAIServing):
338 )
339 choice_data = ChatCompletionResponseStreamChoice(
340 index=i,
327 - delta=DeltaMessage(content=delta_text),
341 + delta=delta_message,
342 logprobs=logprobs,
343 finish_reason=output.finish_reason,
344 stop_reason=output.stop_reason)
@@ -381,9 +395,22 @@ class OpenAIServingChat(OpenAIServing):
395 else:
396 logprobs = None
397
398 + if request.tool_choice and type(
399 + request.tool_choice) is ChatCompletionNamedToolChoiceParam:
400 + message = ChatMessage(
401 + role=role,
402 + content="",
403 + tool_calls=[
404 + ToolCall(function=FunctionCall(
405 + name=request.tool_choice.function.name,
406 + arguments=output.text))
407 + ])
408 + elif not request.tool_choice or request.tool_choice == "none":
409 + message = ChatMessage(role=role, content=output.text)
410 +
411 choice_data = ChatCompletionResponseChoice(
412 index=output.index,
386 - message=ChatMessage(role=role, content=output.text),
413 + message=message,
414 logprobs=logprobs,
415 finish_reason=output.finish_reason,
416 stop_reason=output.stop_reason)
vllm/model_executor/guided_decoding/__init__.py
+28 -2
@@ -1,7 +1,8 @@
1 from typing import Optional, Union
2
3 -from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
4 - CompletionRequest)
3 +from vllm.entrypoints.openai.protocol import (
4 + ChatCompletionNamedToolChoiceParam, ChatCompletionRequest,
5 + CompletionRequest)
6 from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import (
7 get_lm_format_enforcer_guided_decoding_logits_processor)
8 from vllm.model_executor.guided_decoding.outlines_decoding import (
@@ -13,6 +14,8 @@ async def get_guided_decoding_logits_processor(
14 guided_decoding_backend: str, request: Union[CompletionRequest,
15 ChatCompletionRequest],
16 tokenizer) -> Optional[LogitsProcessor]:
17 + request = _adapt_request_for_tool_use(request)
18 +
19 if guided_decoding_backend == 'outlines':
20 return await get_outlines_guided_decoding_logits_processor(
21 request, tokenizer)
@@ -23,3 +26,26 @@ async def get_guided_decoding_logits_processor(
26 raise ValueError(
27 f"Unknown guided decoding backend '{guided_decoding_backend}'. "
28 "Must be one of 'outlines, 'lm-format-enforcer'")
29 +
30 +
31 +def _adapt_request_for_tool_use(request: Union[CompletionRequest,
32 + ChatCompletionRequest]):
33 + # the legacy completion API does not support tool use
34 + if type(request) is CompletionRequest:
35 + return request
36 +
37 + # user has chosen to not use any tool
38 + if request.tool_choice == "none":
39 + return request
40 +
41 + # user has chosen to use a named tool
42 + if type(request.tool_choice) is ChatCompletionNamedToolChoiceParam:
43 + tool_name = request.tool_choice.function.name
44 + tools = {tool.function.name: tool.function for tool in request.tools}
45 + if tool_name not in tools:
46 + raise ValueError(
47 + f"Tool '{tool_name}' has not been passed in `tools`.")
48 + tool = tools[tool_name]
49 + request.guided_json = tool.parameters
50 +
51 + return request