A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[BugFix] Fix chat API continuous usage stats (#9357)
Nick Hill committed
Oct 15, 2024 at 07:19 UTC
e9d517f27673ec8736c026f2311d3c250d5f9061
2 files changed
+53
-76
tests/entrypoints/openai/test_chat.py
+12
-2
@@ -433,18 +433,28 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
433
model=model_name,
434
messages=messages,
435
max_tokens=10,
436
+ extra_body=dict(min_tokens=10),
437
temperature=0.0,
438
stream=True,
439
stream_options={
440
"include_usage": True,
440
- "continuous_usage_stats": True
441
+ "continuous_usage_stats": True,
442
},
443
)
444
+ last_completion_tokens = 0
445
async for chunk in stream:
446
assert chunk.usage.prompt_tokens >= 0
445
- assert chunk.usage.completion_tokens >= 0
447
+ assert last_completion_tokens == 0 or \
448
+ chunk.usage.completion_tokens > last_completion_tokens or \
449
+ (
450
+ not chunk.choices and
451
+ chunk.usage.completion_tokens == last_completion_tokens
452
+ )
453
assert chunk.usage.total_tokens == (chunk.usage.prompt_tokens +
454
chunk.usage.completion_tokens)
455
+ last_completion_tokens = chunk.usage.completion_tokens
456
+
457
+ assert last_completion_tokens == 10
458
459
460
# NOTE: Not sure why, but when I place this after `test_guided_regex_chat`
vllm/entrypoints/openai/serving_chat.py
+41
-74
@@ -330,6 +330,14 @@ class OpenAIServingChat(OpenAIServing):
330
yield "data: [DONE]\n\n"
331
return
332
333
+ stream_options = request.stream_options
334
+ if stream_options:
335
+ include_usage = stream_options.include_usage
336
+ include_continuous_usage = include_usage and \
337
+ stream_options.continuous_usage_stats
338
+ else:
339
+ include_usage, include_continuous_usage = False, False
340
+
341
try:
342
async for res in result_generator:
343
if res.prompt_token_ids is not None:
@@ -348,7 +356,6 @@ class OpenAIServingChat(OpenAIServing):
356
# NOTE num_choices defaults to 1 so this usually executes
357
# once per request
358
for i in range(num_choices):
351
- tool_parser = tool_parsers[i]
359
choice_data = ChatCompletionResponseStreamChoice(
360
index=i,
361
delta=DeltaMessage(
@@ -364,19 +371,12 @@ class OpenAIServingChat(OpenAIServing):
371
choices=[choice_data],
372
model=model_name)
373
367
- # if usage should be included
368
- if (request.stream_options
369
- and request.stream_options.include_usage):
370
- # if continuous usage stats are requested, add it
371
- if request.stream_options.continuous_usage_stats:
372
- usage = UsageInfo(
373
- prompt_tokens=num_prompt_tokens,
374
- completion_tokens=0,
375
- total_tokens=num_prompt_tokens)
376
- chunk.usage = usage
377
- # otherwise don't
378
- else:
379
- chunk.usage = None
374
+ # if continuous usage stats are requested, add it
375
+ if include_continuous_usage:
376
+ chunk.usage = UsageInfo(
377
+ prompt_tokens=num_prompt_tokens,
378
+ completion_tokens=0,
379
+ total_tokens=num_prompt_tokens)
380
381
data = chunk.model_dump_json(exclude_unset=True)
382
yield f"data: {data}\n\n"
@@ -404,17 +404,11 @@ class OpenAIServingChat(OpenAIServing):
404
created=created_time,
405
choices=[choice_data],
406
model=model_name)
407
- if (request.stream_options and
408
- request.stream_options.include_usage):
409
- if (request.stream_options.
410
- continuous_usage_stats):
411
- usage = UsageInfo(
412
- prompt_tokens=num_prompt_tokens,
413
- completion_tokens=0,
414
- total_tokens=num_prompt_tokens)
415
- chunk.usage = usage
416
- else:
417
- chunk.usage = None
407
+ if include_continuous_usage:
408
+ chunk.usage = UsageInfo(
409
+ prompt_tokens=num_prompt_tokens,
410
+ completion_tokens=0,
411
+ total_tokens=num_prompt_tokens)
412
413
data = chunk.model_dump_json(
414
exclude_unset=True)
@@ -494,36 +488,11 @@ class OpenAIServingChat(OpenAIServing):
488
489
if output.finish_reason is None:
490
# Send token-by-token response for each request.n
497
-
491
choice_data = ChatCompletionResponseStreamChoice(
492
index=i,
493
delta=delta_message,
494
logprobs=logprobs,
495
finish_reason=None)
503
- chunk = ChatCompletionStreamResponse(
504
- id=request_id,
505
- object=chunk_object_type,
506
- created=created_time,
507
- choices=[choice_data],
508
- model=model_name)
509
-
510
- # handle usage stats if requested & if continuous
511
- if (request.stream_options
512
- and request.stream_options.include_usage):
513
- if request.stream_options.continuous_usage_stats:
514
- completion_tokens = len(output.token_ids)
515
- usage = UsageInfo(
516
- prompt_tokens=num_prompt_tokens,
517
- completion_tokens=completion_tokens,
518
- total_tokens=num_prompt_tokens +
519
- completion_tokens,
520
- )
521
- chunk.usage = usage
522
- else:
523
- chunk.usage = None
524
-
525
- data = chunk.model_dump_json(exclude_unset=True)
526
- yield f"data: {data}\n\n"
496
497
# if the model is finished generating
498
else:
@@ -573,34 +542,32 @@ class OpenAIServingChat(OpenAIServing):
542
finish_reason=output.finish_reason
543
if not auto_tools_called else "tool_calls",
544
stop_reason=output.stop_reason)
576
- chunk = ChatCompletionStreamResponse(
577
- id=request_id,
578
- object=chunk_object_type,
579
- created=created_time,
580
- choices=[choice_data],
581
- model=model_name)
582
- if (request.stream_options
583
- and request.stream_options.include_usage):
584
- if request.stream_options.continuous_usage_stats:
585
- completion_tokens = len(output.token_ids)
586
- usage = UsageInfo(
587
- prompt_tokens=num_prompt_tokens,
588
- completion_tokens=completion_tokens,
589
- total_tokens=num_prompt_tokens +
590
- completion_tokens,
591
- )
592
- chunk.usage = usage
593
- else:
594
- chunk.usage = None
595
- data = chunk.model_dump_json(exclude_unset=True)
596
- yield f"data: {data}\n\n"
545
+
546
finish_reason_sent[i] = True
547
548
+ chunk = ChatCompletionStreamResponse(
549
+ id=request_id,
550
+ object=chunk_object_type,
551
+ created=created_time,
552
+ choices=[choice_data],
553
+ model=model_name)
554
+
555
+ # handle usage stats if requested & if continuous
556
+ if include_continuous_usage:
557
+ completion_tokens = previous_num_tokens[i]
558
+ chunk.usage = UsageInfo(
559
+ prompt_tokens=num_prompt_tokens,
560
+ completion_tokens=completion_tokens,
561
+ total_tokens=num_prompt_tokens + completion_tokens,
562
+ )
563
+
564
+ data = chunk.model_dump_json(exclude_unset=True)
565
+ yield f"data: {data}\n\n"
566
+
567
# once the final token is handled, if stream_options.include_usage
568
# is sent, send the usage
601
- if (request.stream_options
602
- and request.stream_options.include_usage):
603
- completion_tokens = previous_num_tokens[i]
569
+ if include_usage:
570
+ completion_tokens = sum(previous_num_tokens)
571
final_usage = UsageInfo(
572
prompt_tokens=num_prompt_tokens,
573
completion_tokens=completion_tokens,