A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Return usage for openai streaming requests (#1663)
Iskren Ivov Chernev committed
Nov 17, 2023 at 01:28 UTC
686f5e321096f3a843a23a89fa4a3f28e586c26f
2 files changed
+29
-3
vllm/entrypoints/openai/api_server.py
+26
-3
@@ -245,6 +245,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
245
index: int,
246
text: str,
247
finish_reason: Optional[str] = None,
248
+ usage: Optional[UsageInfo] = None,
249
) -> str:
250
choice_data = ChatCompletionResponseStreamChoice(
251
index=index,
@@ -257,7 +258,10 @@ async def create_chat_completion(request: ChatCompletionRequest,
258
model=model_name,
259
choices=[choice_data],
260
)
260
- response_json = response.json(ensure_ascii=False)
261
+ if usage is not None:
262
+ response.usage = usage
263
+ # exclude unset to leave details out of each sse
264
+ response_json = response.json(exclude_unset=True, ensure_ascii=False)
265
266
return response_json
267
@@ -283,17 +287,25 @@ async def create_chat_completion(request: ChatCompletionRequest,
287
i = output.index
288
delta_text = output.text[len(previous_texts[i]):]
289
previous_texts[i] = output.text
286
- previous_num_tokens[i] = len(output.token_ids)
290
+ completion_tokens = len(output.token_ids)
291
+ previous_num_tokens[i] = completion_tokens
292
response_json = create_stream_response_json(
293
index=i,
294
text=delta_text,
295
)
296
yield f"data: {response_json}\n\n"
297
if output.finish_reason is not None:
298
+ prompt_tokens = len(res.prompt_token_ids)
299
+ final_usage = UsageInfo(
300
+ prompt_tokens=prompt_tokens,
301
+ completion_tokens=completion_tokens,
302
+ total_tokens=prompt_tokens + completion_tokens,
303
+ )
304
response_json = create_stream_response_json(
305
index=i,
306
text="",
307
finish_reason=output.finish_reason,
308
+ usage=final_usage,
309
)
310
yield f"data: {response_json}\n\n"
311
yield "data: [DONE]\n\n"
@@ -462,6 +474,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
474
text: str,
475
logprobs: Optional[LogProbs] = None,
476
finish_reason: Optional[str] = None,
477
+ usage: Optional[UsageInfo] = None,
478
) -> str:
479
choice_data = CompletionResponseStreamChoice(
480
index=index,
@@ -475,7 +488,9 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
488
model=model_name,
489
choices=[choice_data],
490
)
478
- response_json = response.json(ensure_ascii=False)
491
+ if usage is not None:
492
+ response.usage = usage
493
+ response_json = response.json(exclude_unset=True, ensure_ascii=False)
494
495
return response_json
496
@@ -505,11 +520,19 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
520
if output.finish_reason is not None:
521
logprobs = (LogProbs()
522
if request.logprobs is not None else None)
523
+ prompt_tokens = len(res.prompt_token_ids)
524
+ completion_tokens = len(output.token_ids)
525
+ final_usage = UsageInfo(
526
+ prompt_tokens=prompt_tokens,
527
+ completion_tokens=completion_tokens,
528
+ total_tokens=prompt_tokens + completion_tokens,
529
+ )
530
response_json = create_stream_response_json(
531
index=i,
532
text="",
533
logprobs=logprobs,
534
finish_reason=output.finish_reason,
535
+ usage=final_usage,
536
)
537
yield f"data: {response_json}\n\n"
538
yield "data: [DONE]\n\n"
vllm/entrypoints/openai/protocol.py
+3
@@ -139,6 +139,7 @@ class CompletionStreamResponse(BaseModel):
139
created: int = Field(default_factory=lambda: int(time.time()))
140
model: str
141
choices: List[CompletionResponseStreamChoice]
142
+ usage: Optional[UsageInfo]
143
144
145
class ChatMessage(BaseModel):
@@ -178,3 +179,5 @@ class ChatCompletionStreamResponse(BaseModel):
179
created: int = Field(default_factory=lambda: int(time.time()))
180
model: str
181
choices: List[ChatCompletionResponseStreamChoice]
182
+ usage: Optional[UsageInfo] = Field(
183
+ default=None, description="data about request and response")