@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e9d517f276
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[BugFix] Fix chat API continuous usage stats (#9357)

Nick Hill committed Oct 15, 2024 at 07:19 UTC e9d517f27673ec8736c026f2311d3c250d5f9061
2 files changed +53 -76
tests/entrypoints/openai/test_chat.py
+12 -2
@@ -433,18 +433,28 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,
433 model=model_name,
434 messages=messages,
435 max_tokens=10,
436 + extra_body=dict(min_tokens=10),
437 temperature=0.0,
438 stream=True,
439 stream_options={
440 "include_usage": True,
440 - "continuous_usage_stats": True
441 + "continuous_usage_stats": True,
442 },
443 )
444 + last_completion_tokens = 0
445 async for chunk in stream:
446 assert chunk.usage.prompt_tokens >= 0
445 - assert chunk.usage.completion_tokens >= 0
447 + assert last_completion_tokens == 0 or \
448 + chunk.usage.completion_tokens > last_completion_tokens or \
449 + (
450 + not chunk.choices and
451 + chunk.usage.completion_tokens == last_completion_tokens
452 + )
453 assert chunk.usage.total_tokens == (chunk.usage.prompt_tokens +
454 chunk.usage.completion_tokens)
455 + last_completion_tokens = chunk.usage.completion_tokens
456 +
457 + assert last_completion_tokens == 10
458
459
460 # NOTE: Not sure why, but when I place this after `test_guided_regex_chat`
vllm/entrypoints/openai/serving_chat.py
+41 -74
@@ -330,6 +330,14 @@ class OpenAIServingChat(OpenAIServing):
330 yield "data: [DONE]\n\n"
331 return
332
333 + stream_options = request.stream_options
334 + if stream_options:
335 + include_usage = stream_options.include_usage
336 + include_continuous_usage = include_usage and \
337 + stream_options.continuous_usage_stats
338 + else:
339 + include_usage, include_continuous_usage = False, False
340 +
341 try:
342 async for res in result_generator:
343 if res.prompt_token_ids is not None:
@@ -348,7 +356,6 @@ class OpenAIServingChat(OpenAIServing):
356 # NOTE num_choices defaults to 1 so this usually executes
357 # once per request
358 for i in range(num_choices):
351 - tool_parser = tool_parsers[i]
359 choice_data = ChatCompletionResponseStreamChoice(
360 index=i,
361 delta=DeltaMessage(
@@ -364,19 +371,12 @@ class OpenAIServingChat(OpenAIServing):
371 choices=[choice_data],
372 model=model_name)
373
367 - # if usage should be included
368 - if (request.stream_options
369 - and request.stream_options.include_usage):
370 - # if continuous usage stats are requested, add it
371 - if request.stream_options.continuous_usage_stats:
372 - usage = UsageInfo(
373 - prompt_tokens=num_prompt_tokens,
374 - completion_tokens=0,
375 - total_tokens=num_prompt_tokens)
376 - chunk.usage = usage
377 - # otherwise don't
378 - else:
379 - chunk.usage = None
374 + # if continuous usage stats are requested, add it
375 + if include_continuous_usage:
376 + chunk.usage = UsageInfo(
377 + prompt_tokens=num_prompt_tokens,
378 + completion_tokens=0,
379 + total_tokens=num_prompt_tokens)
380
381 data = chunk.model_dump_json(exclude_unset=True)
382 yield f"data: {data}\n\n"
@@ -404,17 +404,11 @@ class OpenAIServingChat(OpenAIServing):
404 created=created_time,
405 choices=[choice_data],
406 model=model_name)
407 - if (request.stream_options and
408 - request.stream_options.include_usage):
409 - if (request.stream_options.
410 - continuous_usage_stats):
411 - usage = UsageInfo(
412 - prompt_tokens=num_prompt_tokens,
413 - completion_tokens=0,
414 - total_tokens=num_prompt_tokens)
415 - chunk.usage = usage
416 - else:
417 - chunk.usage = None
407 + if include_continuous_usage:
408 + chunk.usage = UsageInfo(
409 + prompt_tokens=num_prompt_tokens,
410 + completion_tokens=0,
411 + total_tokens=num_prompt_tokens)
412
413 data = chunk.model_dump_json(
414 exclude_unset=True)
@@ -494,36 +488,11 @@ class OpenAIServingChat(OpenAIServing):
488
489 if output.finish_reason is None:
490 # Send token-by-token response for each request.n
497 -
491 choice_data = ChatCompletionResponseStreamChoice(
492 index=i,
493 delta=delta_message,
494 logprobs=logprobs,
495 finish_reason=None)
503 - chunk = ChatCompletionStreamResponse(
504 - id=request_id,
505 - object=chunk_object_type,
506 - created=created_time,
507 - choices=[choice_data],
508 - model=model_name)
509 -
510 - # handle usage stats if requested & if continuous
511 - if (request.stream_options
512 - and request.stream_options.include_usage):
513 - if request.stream_options.continuous_usage_stats:
514 - completion_tokens = len(output.token_ids)
515 - usage = UsageInfo(
516 - prompt_tokens=num_prompt_tokens,
517 - completion_tokens=completion_tokens,
518 - total_tokens=num_prompt_tokens +
519 - completion_tokens,
520 - )
521 - chunk.usage = usage
522 - else:
523 - chunk.usage = None
524 -
525 - data = chunk.model_dump_json(exclude_unset=True)
526 - yield f"data: {data}\n\n"
496
497 # if the model is finished generating
498 else:
@@ -573,34 +542,32 @@ class OpenAIServingChat(OpenAIServing):
542 finish_reason=output.finish_reason
543 if not auto_tools_called else "tool_calls",
544 stop_reason=output.stop_reason)
576 - chunk = ChatCompletionStreamResponse(
577 - id=request_id,
578 - object=chunk_object_type,
579 - created=created_time,
580 - choices=[choice_data],
581 - model=model_name)
582 - if (request.stream_options
583 - and request.stream_options.include_usage):
584 - if request.stream_options.continuous_usage_stats:
585 - completion_tokens = len(output.token_ids)
586 - usage = UsageInfo(
587 - prompt_tokens=num_prompt_tokens,
588 - completion_tokens=completion_tokens,
589 - total_tokens=num_prompt_tokens +
590 - completion_tokens,
591 - )
592 - chunk.usage = usage
593 - else:
594 - chunk.usage = None
595 - data = chunk.model_dump_json(exclude_unset=True)
596 - yield f"data: {data}\n\n"
545 +
546 finish_reason_sent[i] = True
547
548 + chunk = ChatCompletionStreamResponse(
549 + id=request_id,
550 + object=chunk_object_type,
551 + created=created_time,
552 + choices=[choice_data],
553 + model=model_name)
554 +
555 + # handle usage stats if requested & if continuous
556 + if include_continuous_usage:
557 + completion_tokens = previous_num_tokens[i]
558 + chunk.usage = UsageInfo(
559 + prompt_tokens=num_prompt_tokens,
560 + completion_tokens=completion_tokens,
561 + total_tokens=num_prompt_tokens + completion_tokens,
562 + )
563 +
564 + data = chunk.model_dump_json(exclude_unset=True)
565 + yield f"data: {data}\n\n"
566 +
567 # once the final token is handled, if stream_options.include_usage
568 # is sent, send the usage
601 - if (request.stream_options
602 - and request.stream_options.include_usage):
603 - completion_tokens = previous_num_tokens[i]
569 + if include_usage:
570 + completion_tokens = sum(previous_num_tokens)
571 final_usage = UsageInfo(
572 prompt_tokens=num_prompt_tokens,
573 completion_tokens=completion_tokens,