A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fix: `AttributeError` in OpenAI-compatible server (#3018)
Jae-Won Chung committed
Feb 29, 2024 at 01:04 UTC
a6d471c75939b2f4708a4e1cb1aa3b7b993ee54b
2 files changed
+2
-2
vllm/entrypoints/openai/protocol.py
+1
-1
@@ -57,7 +57,7 @@ class UsageInfo(BaseModel):
57
58
class ChatCompletionRequest(BaseModel):
59
model: str
60
- messages: Union[str, List[Dict[str, str]]]
60
+ messages: List[Dict[str, str]]
61
temperature: Optional[float] = 0.7
62
top_p: Optional[float] = 1.0
63
n: Optional[int] = 1
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -80,7 +80,7 @@ class OpenAIServingChat(OpenAIServing):
80
if request.add_generation_prompt:
81
return self.response_role
82
else:
83
- return request.messages[-1].role
83
+ return request.messages[-1]["role"]
84
85
async def chat_completion_stream_generator(
86
self, request: ChatCompletionRequest,