A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Avoid list copies in `serving_chat.py` (#22947)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Aug 15, 2025 at 19:06 UTC
f6b5040590e2ca986e6221d98b736a71896eaa53
2 files changed
+16
-15
vllm/entrypoints/openai/serving_chat.py
+15
-14
@@ -50,6 +50,7 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
50
from vllm.transformers_utils.tokenizers import (maybe_serialize_tool_calls,
51
truncate_tool_call_ids,
52
validate_request_params)
53
+from vllm.utils import as_list
54
55
logger = init_logger(__name__)
56
@@ -670,10 +671,10 @@ class OpenAIServingChat(OpenAIServing):
671
672
# avoid the None + list error.
673
if previous_token_ids:
673
- current_token_ids = previous_token_ids + list(
674
+ current_token_ids = previous_token_ids + as_list(
675
output.token_ids)
676
else:
676
- current_token_ids = list(output.token_ids)
677
+ current_token_ids = as_list(output.token_ids)
678
679
if self.use_harmony:
680
if is_final:
@@ -703,11 +704,10 @@ class OpenAIServingChat(OpenAIServing):
704
# set reasoning status to end.
705
# Only keep 'content', remove 'reasoning_content'.
706
if reasoning_parser.is_reasoning_end(
706
- list(output.token_ids)) or \
707
- (res.prompt_token_ids and
708
- reasoning_parser.is_reasoning_end(
709
- list(res.prompt_token_ids)
710
- )):
707
+ as_list(output.token_ids)) or (
708
+ res.prompt_token_ids
709
+ and reasoning_parser.is_reasoning_end(
710
+ res.prompt_token_ids)):
711
reasoning_end_arr[i] = True
712
if delta_message and delta_message.content:
713
# This need to be added to next `delta_text`
@@ -771,6 +771,7 @@ class OpenAIServingChat(OpenAIServing):
771
assert reasoning_parser is not None
772
assert added_content_delta_arr is not None
773
assert reasoning_end_arr is not None
774
+ output_token_ids = as_list(output.token_ids)
775
if not reasoning_end_arr[i]:
776
delta_message = (
777
reasoning_parser.
@@ -780,7 +781,7 @@ class OpenAIServingChat(OpenAIServing):
781
delta_text,
782
previous_token_ids,
783
current_token_ids,
783
- output.token_ids,
784
+ output_token_ids,
785
))
786
# When encountering think end id in prompt_token_ids
787
# i.e {"enable_thinking": False},
@@ -789,9 +790,9 @@ class OpenAIServingChat(OpenAIServing):
790
# to 'reasoning_content'.
791
if res.prompt_token_ids and \
792
reasoning_parser.is_reasoning_end(
792
- list(res.prompt_token_ids)):
793
+ res.prompt_token_ids):
794
reasoning_end_arr[i] = True
794
- current_token_ids = list(output.token_ids)
795
+ current_token_ids = output_token_ids
796
if delta_message and delta_message.content:
797
current_text = delta_message.content
798
delta_message.content = None
@@ -802,11 +803,11 @@ class OpenAIServingChat(OpenAIServing):
803
# Remove the text and token ids related
804
# to 'reasoning_content'.
805
if reasoning_parser.is_reasoning_end(
805
- list(output.token_ids)):
806
+ output_token_ids):
807
reasoning_end_arr[i] = True
808
current_token_ids = \
809
reasoning_parser.extract_content_ids(
809
- list(output.token_ids))
810
+ output_token_ids)
811
if delta_message and delta_message.content:
812
current_text = delta_message.content
813
delta_message.content = None
@@ -815,7 +816,7 @@ class OpenAIServingChat(OpenAIServing):
816
817
# handle tool calls only after reasoning is done,
818
else:
818
- delta_token_ids = list(output.token_ids)
819
+ delta_token_ids = output_token_ids
820
# First time to tool call,
821
# add the remaining text and token ids
822
# to delta from previous
@@ -899,7 +900,7 @@ class OpenAIServingChat(OpenAIServing):
900
self.request_logger.log_outputs(
901
request_id=request_id,
902
outputs=delta_content,
902
- output_token_ids=list(output.token_ids),
903
+ output_token_ids=as_list(output.token_ids),
904
finish_reason=output.finish_reason,
905
is_streaming=True,
906
delta=True,
vllm/reasoning/abs_reasoning_parsers.py
+1
-1
@@ -44,7 +44,7 @@ class ReasoningParser:
44
return self.model_tokenizer.get_vocab()
45
46
@abstractmethod
47
- def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
47
+ def is_reasoning_end(self, input_ids: list[int]) -> bool:
48
"""
49
Check if the reasoning content ends in the input_ids.
50