A high-throughput and memory-efficient inference and serving engine for LLMs
[Refactor] Remove dead states from chat completion serving (#45081)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Jun 10, 2026 at 01:20 UTC
6aec99f030feeea083bd63f6931f39a99a408533
1 file changed
+1
-73
vllm/entrypoints/openai/chat_completion/serving.py
+1
-73
@@ -376,7 +376,6 @@ class OpenAIServingChat(OpenAIServing):
376
conversation,
377
tokenizer,
378
request_metadata,
379
- reasoning_parser,
379
chat_template_kwargs=chat_template_kwargs,
380
)
381
@@ -405,7 +404,6 @@ class OpenAIServingChat(OpenAIServing):
404
conversation: list[ConversationMessage],
405
tokenizer: TokenizerLike,
406
request_metadata: RequestResponseMetadata,
408
- reasoning_parser: ReasoningParser | None = None,
407
chat_template_kwargs: dict[str, Any] | None = None,
408
) -> AsyncGenerator[str, None]:
409
created_time = int(time.time())
@@ -430,33 +428,13 @@ class OpenAIServingChat(OpenAIServing):
428
else:
429
tool_choice_function_name = None
430
433
- # Determine whether tools are in use with "auto" tool choice
434
- tool_choice_auto = (
435
- not tool_choice_function_name
436
- and self._should_stream_with_auto_tool_parsing(request)
437
- )
438
-
439
- all_previous_token_ids: list[list[int]] | None
431
if self.tool_call_id_type == "kimi_k2":
432
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
433
else:
434
history_tool_call_cnt = 0
435
445
- # Always track previous_texts for comprehensive output logging
436
previous_texts = [""] * num_choices
437
448
- # Only one of these will be used, thus previous_texts and
449
- # all_previous_token_ids will not be used twice in the same iteration.
450
- if (
451
- tool_choice_auto
452
- or tool_choice_function_name
453
- or request.tool_choice == "required"
454
- or reasoning_parser
455
- ):
456
- all_previous_token_ids = [[] for _ in range(num_choices)]
457
- else:
458
- all_previous_token_ids = None
459
-
438
try:
439
if self.parser_cls is not None:
440
if tokenizer is None:
@@ -639,26 +617,6 @@ class OpenAIServingChat(OpenAIServing):
617
618
delta_message: DeltaMessage | None
619
642
- # just update previous_texts and previous_token_ids
643
- if (
644
- tool_choice_auto
645
- or tool_choice_function_name
646
- or request.tool_choice == "required"
647
- or reasoning_parser
648
- ):
649
- assert previous_texts is not None
650
- assert all_previous_token_ids is not None
651
- previous_text = previous_texts[i]
652
- previous_token_ids = all_previous_token_ids[i]
653
- current_text = previous_text + delta_text
654
- # avoid the None + list error.
655
- if previous_token_ids:
656
- current_token_ids = previous_token_ids + as_list(
657
- output.token_ids
658
- )
659
- else:
660
- current_token_ids = as_list(output.token_ids)
661
-
620
if self.use_harmony:
621
delta_message, tools_streamed_flag = (
622
extract_harmony_streaming_delta(
@@ -683,21 +641,7 @@ class OpenAIServingChat(OpenAIServing):
641
else:
642
delta_message = DeltaMessage(content=delta_text)
643
686
- # update the previous values for the next iteration
687
- if (
688
- tool_choice_auto
689
- or tool_choice_function_name
690
- or request.tool_choice == "required"
691
- or reasoning_parser
692
- ) and not self.use_harmony:
693
- assert previous_texts is not None
694
- assert all_previous_token_ids is not None
695
- previous_texts[i] = current_text
696
- all_previous_token_ids[i] = current_token_ids
697
- else:
698
- # Update for comprehensive logging even in simple case
699
- assert previous_texts is not None
700
- previous_texts[i] += delta_text
644
+ previous_texts[i] += delta_text
645
646
# set the previous values for the next iteration
647
previous_num_tokens[i] += len(output.token_ids)
@@ -1326,19 +1270,3 @@ class OpenAIServingChat(OpenAIServing):
1270
)
1271
1272
return ChatCompletionLogProbs(content=logprobs_content)
1329
-
1330
- def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1331
- """
1332
- Utility function to check if streamed tokens should go through the tool
1333
- call parser that was configured.
1334
-
1335
- We only want to do this IF user-provided tools are set, a tool parser
1336
- is configured, "auto" tool choice is enabled, and the request's tool
1337
- choice field indicates that "auto" tool choice should be used.
1338
- """
1339
- return (
1340
- request.tools
1341
- and self.tool_parser
1342
- and self.enable_auto_tools
1343
- and request.tool_choice in ["auto", None]
1344
- )