@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fc0d9dfc3a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Frontend] Re-enable custom roles in Chat Completions API (#4758)

Cyrus Leung committed May 16, 2024 at 05:58 UTC fc0d9dfc3afcea2e23649ef8eb8bbe0446682813
3 files changed +108 -26
tests/entrypoints/test_openai_server.py
+30
@@ -783,6 +783,36 @@ async def test_complex_message_content(server, client: openai.AsyncOpenAI):
783 assert content == "2"
784
785
786 +async def test_custom_role(server, client: openai.AsyncOpenAI):
787 + # Not sure how the model handles custom roles so we just check that
788 + # both string and complex message content are handled in the same way
789 +
790 + resp1 = await client.chat.completions.create(
791 + model=MODEL_NAME,
792 + messages=[{
793 + "role": "my-custom-role",
794 + "content": "what is 1+1?",
795 + }], # type: ignore
796 + temperature=0,
797 + seed=0)
798 +
799 + resp2 = await client.chat.completions.create(
800 + model=MODEL_NAME,
801 + messages=[{
802 + "role": "my-custom-role",
803 + "content": [{
804 + "type": "text",
805 + "text": "what is 1+1?"
806 + }]
807 + }], # type: ignore
808 + temperature=0,
809 + seed=0)
810 +
811 + content1 = resp1.choices[0].message.content
812 + content2 = resp2.choices[0].message.content
813 + assert content1 == content2
814 +
815 +
816 async def test_guided_grammar(server, client: openai.AsyncOpenAI):
817 simple_sql_grammar = """
818 start: select_statement
vllm/entrypoints/openai/protocol.py
+36 -2
@@ -3,16 +3,50 @@
3 import time
4 from typing import Any, Dict, List, Literal, Optional, Union
5
6 +import openai.types.chat
7 import torch
7 -from openai.types.chat import ChatCompletionMessageParam
8 from pydantic import BaseModel, ConfigDict, Field, model_validator
9 -from typing_extensions import Annotated
9 +# pydantic needs the TypedDict from typing_extensions
10 +from typing_extensions import Annotated, Required, TypedDict
11
12 from vllm.pooling_params import PoolingParams
13 from vllm.sampling_params import SamplingParams
14 from vllm.utils import random_uuid
15
16
17 +class CustomChatCompletionContentPartParam(TypedDict, total=False):
18 + __pydantic_config__ = ConfigDict(extra="allow") # type: ignore
19 +
20 + type: Required[str]
21 + """The type of the content part."""
22 +
23 +
24 +ChatCompletionContentPartParam = Union[
25 + openai.types.chat.ChatCompletionContentPartParam,
26 + CustomChatCompletionContentPartParam]
27 +
28 +
29 +class CustomChatCompletionMessageParam(TypedDict, total=False):
30 + """Enables custom roles in the Chat Completion API."""
31 + role: Required[str]
32 + """The role of the message's author."""
33 +
34 + content: Union[str, List[ChatCompletionContentPartParam]]
35 + """The contents of the message."""
36 +
37 + name: str
38 + """An optional name for the participant.
39 +
40 + Provides the model information to differentiate between participants of the
41 + same role.
42 + """
43 +
44 +
45 +ChatCompletionMessageParam = Union[
46 + openai.types.chat.ChatCompletionMessageParam,
47 + CustomChatCompletionMessageParam]
48 +
49 +
50 class OpenAIBaseModel(BaseModel):
51 # OpenAI API does not allow extra fields
52 model_config = ConfigDict(extra="forbid")
vllm/entrypoints/openai/serving_chat.py
+42 -24
@@ -1,15 +1,16 @@
1 import codecs
2 import time
3 -from typing import (AsyncGenerator, AsyncIterator, Awaitable, Iterable, List,
4 - Optional, Tuple, TypedDict, Union, final)
3 +from dataclasses import dataclass
4 +from typing import (AsyncGenerator, AsyncIterator, Iterable, List, Optional,
5 + TypedDict, Union, cast, final)
6
7 from fastapi import Request
7 -from openai.types.chat import (ChatCompletionContentPartParam,
8 - ChatCompletionRole)
8 +from openai.types.chat import ChatCompletionContentPartTextParam
9
10 from vllm.config import ModelConfig
11 from vllm.engine.async_llm_engine import AsyncLLMEngine
12 from vllm.entrypoints.openai.protocol import (
13 + ChatCompletionContentPartParam, ChatCompletionMessageParam,
14 ChatCompletionRequest, ChatCompletionResponse,
15 ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
16 ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
@@ -31,6 +32,11 @@ class ConversationMessage(TypedDict):
32 content: str
33
34
35 +@dataclass(frozen=True)
36 +class ChatMessageParseResult:
37 + messages: List[ConversationMessage]
38 +
39 +
40 class OpenAIServingChat(OpenAIServing):
41
42 def __init__(self,
@@ -77,27 +83,40 @@ class OpenAIServingChat(OpenAIServing):
83 logger.warning(
84 "No chat template provided. Chat API will not work.")
85
80 - def _parse_chat_message_content(
86 + def _parse_chat_message_content_parts(
87 self,
82 - role: ChatCompletionRole,
83 - content: Optional[Union[str,
84 - Iterable[ChatCompletionContentPartParam]]],
85 - ) -> Tuple[List[ConversationMessage], List[Awaitable[object]]]:
86 - if content is None:
87 - return [], []
88 - if isinstance(content, str):
89 - return [ConversationMessage(role=role, content=content)], []
90 -
88 + role: str,
89 + parts: Iterable[ChatCompletionContentPartParam],
90 + ) -> ChatMessageParseResult:
91 texts: List[str] = []
92 - for _, part in enumerate(content):
93 - if part["type"] == "text":
94 - text = part["text"]
92 +
93 + for _, part in enumerate(parts):
94 + part_type = part["type"]
95 + if part_type == "text":
96 + text = cast(ChatCompletionContentPartTextParam, part)["text"]
97
98 texts.append(text)
99 else:
98 - raise NotImplementedError(f"Unknown part type: {part['type']}")
100 + raise NotImplementedError(f"Unknown part type: {part_type}")
101 +
102 + messages = [ConversationMessage(role=role, content="\n".join(texts))]
103 +
104 + return ChatMessageParseResult(messages=messages)
105 +
106 + def _parse_chat_message_content(
107 + self,
108 + message: ChatCompletionMessageParam,
109 + ) -> ChatMessageParseResult:
110 + role = message["role"]
111 + content = message.get("content")
112 +
113 + if content is None:
114 + return ChatMessageParseResult(messages=[])
115 + if isinstance(content, str):
116 + messages = [ConversationMessage(role=role, content=content)]
117 + return ChatMessageParseResult(messages=messages)
118
100 - return [ConversationMessage(role=role, content="\n".join(texts))], []
119 + return self._parse_chat_message_content_parts(role, content)
120
121 async def create_chat_completion(
122 self, request: ChatCompletionRequest, raw_request: Request
@@ -119,11 +138,10 @@ class OpenAIServingChat(OpenAIServing):
138 try:
139 conversation: List[ConversationMessage] = []
140
122 - for m in request.messages:
123 - messages, _ = self._parse_chat_message_content(
124 - m["role"], m["content"])
141 + for msg in request.messages:
142 + parsed_msg = self._parse_chat_message_content(msg)
143
126 - conversation.extend(messages)
144 + conversation.extend(parsed_msg.messages)
145
146 prompt = self.tokenizer.apply_chat_template(
147 conversation=conversation,
@@ -387,4 +405,4 @@ class OpenAIServingChat(OpenAIServing):
405 usage=usage,
406 )
407
390 - return response
\ No newline at end of file
408 + return response