@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a494140433
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Frontend] Support complex message content for chat completions endpoint (#3467)

Co-authored-by: Lily Liu <lilyliupku@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Florian Greinacher committed May 1, 2024 at 01:28 UTC a494140433be496a0321999955acf7e6387986b3
2 files changed +46 -21
tests/entrypoints/test_openai_server.py
+19
@@ -786,6 +786,25 @@ async def test_extra_fields(server, client: openai.AsyncOpenAI):
786 assert "extra_forbidden" in exc_info.value.message
787
788
789 +async def test_complex_message_content(server, client: openai.AsyncOpenAI):
790 + resp = await client.chat.completions.create(
791 + model=MODEL_NAME,
792 + messages=[{
793 + "role":
794 + "user",
795 + "content": [{
796 + "type":
797 + "text",
798 + "text":
799 + "what is 1+1? please provide the result without any other text."
800 + }]
801 + }],
802 + temperature=0,
803 + seed=0)
804 + content = resp.choices[0].message.content
805 + assert content == "2"
806 +
807 +
808 async def test_guided_grammar(server, client: openai.AsyncOpenAI):
809 simple_sql_grammar = """
810 start: select_statement
vllm/entrypoints/openai/serving_chat.py
+27 -21
@@ -55,9 +55,16 @@ class OpenAIServingChat(OpenAIServing):
55 if isinstance(content, str):
56 return [ConversationMessage(role=role, content=content)], []
57
58 - # To be implemented: https://github.com/vllm-project/vllm/pull/3467
59 - # To be implemented: https://github.com/vllm-project/vllm/pull/4200
60 - raise NotImplementedError("Complex input not supported yet")
58 + texts: List[str] = []
59 + for _, part in enumerate(content):
60 + if part["type"] == "text":
61 + text = part["text"]
62 +
63 + texts.append(text)
64 + else:
65 + raise NotImplementedError(f"Unknown part type: {part['type']}")
66 +
67 + return [ConversationMessage(role=role, content="\n".join(texts))], []
68
69 async def create_chat_completion(
70 self, request: ChatCompletionRequest, raw_request: Request
@@ -122,11 +129,12 @@ class OpenAIServingChat(OpenAIServing):
129 # Streaming response
130 if request.stream:
131 return self.chat_completion_stream_generator(
125 - request, result_generator, request_id)
132 + request, result_generator, request_id, conversation)
133 else:
134 try:
135 return await self.chat_completion_full_generator(
129 - request, raw_request, result_generator, request_id)
136 + request, raw_request, result_generator, request_id,
137 + conversation)
138 except ValueError as e:
139 # TODO: Use a vllm-specific Validation Error
140 return self.create_error_response(str(e))
@@ -139,8 +147,9 @@ class OpenAIServingChat(OpenAIServing):
147
148 async def chat_completion_stream_generator(
149 self, request: ChatCompletionRequest,
142 - result_generator: AsyncIterator[RequestOutput],
143 - request_id: str) -> AsyncGenerator[str, None]:
150 + result_generator: AsyncIterator[RequestOutput], request_id: str,
151 + conversation: List[ConversationMessage]
152 + ) -> AsyncGenerator[str, None]:
153 model_name = self.served_model_names[0]
154 created_time = int(time.time())
155 chunk_object_type = "chat.completion.chunk"
@@ -179,12 +188,10 @@ class OpenAIServingChat(OpenAIServing):
188 # last message
189 if request.echo:
190 last_msg_content = ""
182 - if request.messages and isinstance(
183 - request.messages,
184 - list) and request.messages[-1].get(
185 - "content") and request.messages[-1].get(
186 - "role") == role:
187 - last_msg_content = request.messages[-1]["content"]
191 + if conversation and conversation[-1].get(
192 + "content") and conversation[-1].get(
193 + "role") == role:
194 + last_msg_content = conversation[-1]["content"]
195
196 if last_msg_content:
197 for i in range(request.n):
@@ -279,9 +286,10 @@ class OpenAIServingChat(OpenAIServing):
286 yield "data: [DONE]\n\n"
287
288 async def chat_completion_full_generator(
282 - self, request: ChatCompletionRequest, raw_request: Request,
283 - result_generator: AsyncIterator[RequestOutput],
284 - request_id: str) -> Union[ErrorResponse, ChatCompletionResponse]:
289 + self, request: ChatCompletionRequest, raw_request: Request,
290 + result_generator: AsyncIterator[RequestOutput], request_id: str,
291 + conversation: List[ConversationMessage]
292 + ) -> Union[ErrorResponse, ChatCompletionResponse]:
293
294 model_name = self.served_model_names[0]
295 created_time = int(time.time())
@@ -322,11 +330,9 @@ class OpenAIServingChat(OpenAIServing):
330
331 if request.echo:
332 last_msg_content = ""
325 - if request.messages and isinstance(
326 - request.messages, list) and request.messages[-1].get(
327 - "content") and request.messages[-1].get(
328 - "role") == role:
329 - last_msg_content = request.messages[-1]["content"]
333 + if conversation and conversation[-1].get(
334 + "content") and conversation[-1].get("role") == role:
335 + last_msg_content = conversation[-1]["content"]
336
337 for choice in choices:
338 full_message = last_msg_content + choice.message.content