@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 686f5e3210
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Return usage for openai streaming requests (#1663)

Iskren Ivov Chernev committed Nov 17, 2023 at 01:28 UTC 686f5e321096f3a843a23a89fa4a3f28e586c26f
2 files changed +29 -3
vllm/entrypoints/openai/api_server.py
+26 -3
@@ -245,6 +245,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
245 index: int,
246 text: str,
247 finish_reason: Optional[str] = None,
248 + usage: Optional[UsageInfo] = None,
249 ) -> str:
250 choice_data = ChatCompletionResponseStreamChoice(
251 index=index,
@@ -257,7 +258,10 @@ async def create_chat_completion(request: ChatCompletionRequest,
258 model=model_name,
259 choices=[choice_data],
260 )
260 - response_json = response.json(ensure_ascii=False)
261 + if usage is not None:
262 + response.usage = usage
263 + # exclude unset to leave details out of each sse
264 + response_json = response.json(exclude_unset=True, ensure_ascii=False)
265
266 return response_json
267
@@ -283,17 +287,25 @@ async def create_chat_completion(request: ChatCompletionRequest,
287 i = output.index
288 delta_text = output.text[len(previous_texts[i]):]
289 previous_texts[i] = output.text
286 - previous_num_tokens[i] = len(output.token_ids)
290 + completion_tokens = len(output.token_ids)
291 + previous_num_tokens[i] = completion_tokens
292 response_json = create_stream_response_json(
293 index=i,
294 text=delta_text,
295 )
296 yield f"data: {response_json}\n\n"
297 if output.finish_reason is not None:
298 + prompt_tokens = len(res.prompt_token_ids)
299 + final_usage = UsageInfo(
300 + prompt_tokens=prompt_tokens,
301 + completion_tokens=completion_tokens,
302 + total_tokens=prompt_tokens + completion_tokens,
303 + )
304 response_json = create_stream_response_json(
305 index=i,
306 text="",
307 finish_reason=output.finish_reason,
308 + usage=final_usage,
309 )
310 yield f"data: {response_json}\n\n"
311 yield "data: [DONE]\n\n"
@@ -462,6 +474,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
474 text: str,
475 logprobs: Optional[LogProbs] = None,
476 finish_reason: Optional[str] = None,
477 + usage: Optional[UsageInfo] = None,
478 ) -> str:
479 choice_data = CompletionResponseStreamChoice(
480 index=index,
@@ -475,7 +488,9 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
488 model=model_name,
489 choices=[choice_data],
490 )
478 - response_json = response.json(ensure_ascii=False)
491 + if usage is not None:
492 + response.usage = usage
493 + response_json = response.json(exclude_unset=True, ensure_ascii=False)
494
495 return response_json
496
@@ -505,11 +520,19 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
520 if output.finish_reason is not None:
521 logprobs = (LogProbs()
522 if request.logprobs is not None else None)
523 + prompt_tokens = len(res.prompt_token_ids)
524 + completion_tokens = len(output.token_ids)
525 + final_usage = UsageInfo(
526 + prompt_tokens=prompt_tokens,
527 + completion_tokens=completion_tokens,
528 + total_tokens=prompt_tokens + completion_tokens,
529 + )
530 response_json = create_stream_response_json(
531 index=i,
532 text="",
533 logprobs=logprobs,
534 finish_reason=output.finish_reason,
535 + usage=final_usage,
536 )
537 yield f"data: {response_json}\n\n"
538 yield "data: [DONE]\n\n"
vllm/entrypoints/openai/protocol.py
+3
@@ -139,6 +139,7 @@ class CompletionStreamResponse(BaseModel):
139 created: int = Field(default_factory=lambda: int(time.time()))
140 model: str
141 choices: List[CompletionResponseStreamChoice]
142 + usage: Optional[UsageInfo]
143
144
145 class ChatMessage(BaseModel):
@@ -178,3 +179,5 @@ class ChatCompletionStreamResponse(BaseModel):
179 created: int = Field(default_factory=lambda: int(time.time()))
180 model: str
181 choices: List[ChatCompletionResponseStreamChoice]
182 + usage: Optional[UsageInfo] = Field(
183 + default=None, description="data about request and response")