@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a6d471c759
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Fix: `AttributeError` in OpenAI-compatible server (#3018)

Jae-Won Chung committed Feb 29, 2024 at 01:04 UTC a6d471c75939b2f4708a4e1cb1aa3b7b993ee54b
2 files changed +2 -2
vllm/entrypoints/openai/protocol.py
+1 -1
@@ -57,7 +57,7 @@ class UsageInfo(BaseModel):
57
58 class ChatCompletionRequest(BaseModel):
59 model: str
60 - messages: Union[str, List[Dict[str, str]]]
60 + messages: List[Dict[str, str]]
61 temperature: Optional[float] = 0.7
62 top_p: Optional[float] = 1.0
63 n: Optional[int] = 1
vllm/entrypoints/openai/serving_chat.py
+1 -1
@@ -80,7 +80,7 @@ class OpenAIServingChat(OpenAIServing):
80 if request.add_generation_prompt:
81 return self.response_role
82 else:
83 - return request.messages[-1].role
83 + return request.messages[-1]["role"]
84
85 async def chat_completion_stream_generator(
86 self, request: ChatCompletionRequest,