A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Frontend] Support complex message content for chat completions endpoint (#3467)
Co-authored-by: Lily Liu <lilyliupku@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Florian Greinacher committed
May 1, 2024 at 01:28 UTC
a494140433be496a0321999955acf7e6387986b3
2 files changed
+46
-21
tests/entrypoints/test_openai_server.py
+19
@@ -786,6 +786,25 @@ async def test_extra_fields(server, client: openai.AsyncOpenAI):
786
assert "extra_forbidden" in exc_info.value.message
787
788
789
+async def test_complex_message_content(server, client: openai.AsyncOpenAI):
790
+ resp = await client.chat.completions.create(
791
+ model=MODEL_NAME,
792
+ messages=[{
793
+ "role":
794
+ "user",
795
+ "content": [{
796
+ "type":
797
+ "text",
798
+ "text":
799
+ "what is 1+1? please provide the result without any other text."
800
+ }]
801
+ }],
802
+ temperature=0,
803
+ seed=0)
804
+ content = resp.choices[0].message.content
805
+ assert content == "2"
806
+
807
+
808
async def test_guided_grammar(server, client: openai.AsyncOpenAI):
809
simple_sql_grammar = """
810
start: select_statement
vllm/entrypoints/openai/serving_chat.py
+27
-21
@@ -55,9 +55,16 @@ class OpenAIServingChat(OpenAIServing):
55
if isinstance(content, str):
56
return [ConversationMessage(role=role, content=content)], []
57
58
- # To be implemented: https://github.com/vllm-project/vllm/pull/3467
59
- # To be implemented: https://github.com/vllm-project/vllm/pull/4200
60
- raise NotImplementedError("Complex input not supported yet")
58
+ texts: List[str] = []
59
+ for _, part in enumerate(content):
60
+ if part["type"] == "text":
61
+ text = part["text"]
62
+
63
+ texts.append(text)
64
+ else:
65
+ raise NotImplementedError(f"Unknown part type: {part['type']}")
66
+
67
+ return [ConversationMessage(role=role, content="\n".join(texts))], []
68
69
async def create_chat_completion(
70
self, request: ChatCompletionRequest, raw_request: Request
@@ -122,11 +129,12 @@ class OpenAIServingChat(OpenAIServing):
129
# Streaming response
130
if request.stream:
131
return self.chat_completion_stream_generator(
125
- request, result_generator, request_id)
132
+ request, result_generator, request_id, conversation)
133
else:
134
try:
135
return await self.chat_completion_full_generator(
129
- request, raw_request, result_generator, request_id)
136
+ request, raw_request, result_generator, request_id,
137
+ conversation)
138
except ValueError as e:
139
# TODO: Use a vllm-specific Validation Error
140
return self.create_error_response(str(e))
@@ -139,8 +147,9 @@ class OpenAIServingChat(OpenAIServing):
147
148
async def chat_completion_stream_generator(
149
self, request: ChatCompletionRequest,
142
- result_generator: AsyncIterator[RequestOutput],
143
- request_id: str) -> AsyncGenerator[str, None]:
150
+ result_generator: AsyncIterator[RequestOutput], request_id: str,
151
+ conversation: List[ConversationMessage]
152
+ ) -> AsyncGenerator[str, None]:
153
model_name = self.served_model_names[0]
154
created_time = int(time.time())
155
chunk_object_type = "chat.completion.chunk"
@@ -179,12 +188,10 @@ class OpenAIServingChat(OpenAIServing):
188
# last message
189
if request.echo:
190
last_msg_content = ""
182
- if request.messages and isinstance(
183
- request.messages,
184
- list) and request.messages[-1].get(
185
- "content") and request.messages[-1].get(
186
- "role") == role:
187
- last_msg_content = request.messages[-1]["content"]
191
+ if conversation and conversation[-1].get(
192
+ "content") and conversation[-1].get(
193
+ "role") == role:
194
+ last_msg_content = conversation[-1]["content"]
195
196
if last_msg_content:
197
for i in range(request.n):
@@ -279,9 +286,10 @@ class OpenAIServingChat(OpenAIServing):
286
yield "data: [DONE]\n\n"
287
288
async def chat_completion_full_generator(
282
- self, request: ChatCompletionRequest, raw_request: Request,
283
- result_generator: AsyncIterator[RequestOutput],
284
- request_id: str) -> Union[ErrorResponse, ChatCompletionResponse]:
289
+ self, request: ChatCompletionRequest, raw_request: Request,
290
+ result_generator: AsyncIterator[RequestOutput], request_id: str,
291
+ conversation: List[ConversationMessage]
292
+ ) -> Union[ErrorResponse, ChatCompletionResponse]:
293
294
model_name = self.served_model_names[0]
295
created_time = int(time.time())
@@ -322,11 +330,9 @@ class OpenAIServingChat(OpenAIServing):
330
331
if request.echo:
332
last_msg_content = ""
325
- if request.messages and isinstance(
326
- request.messages, list) and request.messages[-1].get(
327
- "content") and request.messages[-1].get(
328
- "role") == role:
329
- last_msg_content = request.messages[-1]["content"]
333
+ if conversation and conversation[-1].get(
334
+ "content") and conversation[-1].get("role") == role:
335
+ last_msg_content = conversation[-1]["content"]
336
337
for choice in choices:
338
full_message = last_msg_content + choice.message.content