A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Frontend] Re-enable custom roles in Chat Completions API (#4758)
Cyrus Leung committed
May 16, 2024 at 05:58 UTC
fc0d9dfc3afcea2e23649ef8eb8bbe0446682813
3 files changed
+108
-26
tests/entrypoints/test_openai_server.py
+30
@@ -783,6 +783,36 @@ async def test_complex_message_content(server, client: openai.AsyncOpenAI):
783
assert content == "2"
784
785
786
+async def test_custom_role(server, client: openai.AsyncOpenAI):
787
+ # Not sure how the model handles custom roles so we just check that
788
+ # both string and complex message content are handled in the same way
789
+
790
+ resp1 = await client.chat.completions.create(
791
+ model=MODEL_NAME,
792
+ messages=[{
793
+ "role": "my-custom-role",
794
+ "content": "what is 1+1?",
795
+ }], # type: ignore
796
+ temperature=0,
797
+ seed=0)
798
+
799
+ resp2 = await client.chat.completions.create(
800
+ model=MODEL_NAME,
801
+ messages=[{
802
+ "role": "my-custom-role",
803
+ "content": [{
804
+ "type": "text",
805
+ "text": "what is 1+1?"
806
+ }]
807
+ }], # type: ignore
808
+ temperature=0,
809
+ seed=0)
810
+
811
+ content1 = resp1.choices[0].message.content
812
+ content2 = resp2.choices[0].message.content
813
+ assert content1 == content2
814
+
815
+
816
async def test_guided_grammar(server, client: openai.AsyncOpenAI):
817
simple_sql_grammar = """
818
start: select_statement
vllm/entrypoints/openai/protocol.py
+36
-2
@@ -3,16 +3,50 @@
3
import time
4
from typing import Any, Dict, List, Literal, Optional, Union
5
6
+import openai.types.chat
7
import torch
7
-from openai.types.chat import ChatCompletionMessageParam
8
from pydantic import BaseModel, ConfigDict, Field, model_validator
9
-from typing_extensions import Annotated
9
+# pydantic needs the TypedDict from typing_extensions
10
+from typing_extensions import Annotated, Required, TypedDict
11
12
from vllm.pooling_params import PoolingParams
13
from vllm.sampling_params import SamplingParams
14
from vllm.utils import random_uuid
15
16
17
+class CustomChatCompletionContentPartParam(TypedDict, total=False):
18
+ __pydantic_config__ = ConfigDict(extra="allow") # type: ignore
19
+
20
+ type: Required[str]
21
+ """The type of the content part."""
22
+
23
+
24
+ChatCompletionContentPartParam = Union[
25
+ openai.types.chat.ChatCompletionContentPartParam,
26
+ CustomChatCompletionContentPartParam]
27
+
28
+
29
+class CustomChatCompletionMessageParam(TypedDict, total=False):
30
+ """Enables custom roles in the Chat Completion API."""
31
+ role: Required[str]
32
+ """The role of the message's author."""
33
+
34
+ content: Union[str, List[ChatCompletionContentPartParam]]
35
+ """The contents of the message."""
36
+
37
+ name: str
38
+ """An optional name for the participant.
39
+
40
+ Provides the model information to differentiate between participants of the
41
+ same role.
42
+ """
43
+
44
+
45
+ChatCompletionMessageParam = Union[
46
+ openai.types.chat.ChatCompletionMessageParam,
47
+ CustomChatCompletionMessageParam]
48
+
49
+
50
class OpenAIBaseModel(BaseModel):
51
# OpenAI API does not allow extra fields
52
model_config = ConfigDict(extra="forbid")
vllm/entrypoints/openai/serving_chat.py
+42
-24
@@ -1,15 +1,16 @@
1
import codecs
2
import time
3
-from typing import (AsyncGenerator, AsyncIterator, Awaitable, Iterable, List,
4
- Optional, Tuple, TypedDict, Union, final)
3
+from dataclasses import dataclass
4
+from typing import (AsyncGenerator, AsyncIterator, Iterable, List, Optional,
5
+ TypedDict, Union, cast, final)
6
7
from fastapi import Request
7
-from openai.types.chat import (ChatCompletionContentPartParam,
8
- ChatCompletionRole)
8
+from openai.types.chat import ChatCompletionContentPartTextParam
9
10
from vllm.config import ModelConfig
11
from vllm.engine.async_llm_engine import AsyncLLMEngine
12
from vllm.entrypoints.openai.protocol import (
13
+ ChatCompletionContentPartParam, ChatCompletionMessageParam,
14
ChatCompletionRequest, ChatCompletionResponse,
15
ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
16
ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
@@ -31,6 +32,11 @@ class ConversationMessage(TypedDict):
32
content: str
33
34
35
+@dataclass(frozen=True)
36
+class ChatMessageParseResult:
37
+ messages: List[ConversationMessage]
38
+
39
+
40
class OpenAIServingChat(OpenAIServing):
41
42
def __init__(self,
@@ -77,27 +83,40 @@ class OpenAIServingChat(OpenAIServing):
83
logger.warning(
84
"No chat template provided. Chat API will not work.")
85
80
- def _parse_chat_message_content(
86
+ def _parse_chat_message_content_parts(
87
self,
82
- role: ChatCompletionRole,
83
- content: Optional[Union[str,
84
- Iterable[ChatCompletionContentPartParam]]],
85
- ) -> Tuple[List[ConversationMessage], List[Awaitable[object]]]:
86
- if content is None:
87
- return [], []
88
- if isinstance(content, str):
89
- return [ConversationMessage(role=role, content=content)], []
90
-
88
+ role: str,
89
+ parts: Iterable[ChatCompletionContentPartParam],
90
+ ) -> ChatMessageParseResult:
91
texts: List[str] = []
92
- for _, part in enumerate(content):
93
- if part["type"] == "text":
94
- text = part["text"]
92
+
93
+ for _, part in enumerate(parts):
94
+ part_type = part["type"]
95
+ if part_type == "text":
96
+ text = cast(ChatCompletionContentPartTextParam, part)["text"]
97
98
texts.append(text)
99
else:
98
- raise NotImplementedError(f"Unknown part type: {part['type']}")
100
+ raise NotImplementedError(f"Unknown part type: {part_type}")
101
+
102
+ messages = [ConversationMessage(role=role, content="\n".join(texts))]
103
+
104
+ return ChatMessageParseResult(messages=messages)
105
+
106
+ def _parse_chat_message_content(
107
+ self,
108
+ message: ChatCompletionMessageParam,
109
+ ) -> ChatMessageParseResult:
110
+ role = message["role"]
111
+ content = message.get("content")
112
+
113
+ if content is None:
114
+ return ChatMessageParseResult(messages=[])
115
+ if isinstance(content, str):
116
+ messages = [ConversationMessage(role=role, content=content)]
117
+ return ChatMessageParseResult(messages=messages)
118
100
- return [ConversationMessage(role=role, content="\n".join(texts))], []
119
+ return self._parse_chat_message_content_parts(role, content)
120
121
async def create_chat_completion(
122
self, request: ChatCompletionRequest, raw_request: Request
@@ -119,11 +138,10 @@ class OpenAIServingChat(OpenAIServing):
138
try:
139
conversation: List[ConversationMessage] = []
140
122
- for m in request.messages:
123
- messages, _ = self._parse_chat_message_content(
124
- m["role"], m["content"])
141
+ for msg in request.messages:
142
+ parsed_msg = self._parse_chat_message_content(msg)
143
126
- conversation.extend(messages)
144
+ conversation.extend(parsed_msg.messages)
145
146
prompt = self.tokenizer.apply_chat_template(
147
conversation=conversation,
@@ -387,4 +405,4 @@ class OpenAIServingChat(OpenAIServing):
405
usage=usage,
406
)
407
390
- return response
\ No newline at end of file
408
+ return response