[Bugfix][Frontend] Keep logprobs of parser-suppressed streaming chunks (#58583)
Signed-off-by: errmakov <ide404@gmail.com> Co-authored-by: Yanxiao Zhao <39199723+sdpkjc@users.noreply.github.com> Co-authored-by: Prakhar Agarwal <270064960+agarwalprakhar2511@users.noreply.github.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
Denis Ermakov committed
Sep 25, 2026 at 18:33 UTC
f339e0e7571c0bd49d6d5f1acb9f21a35b65e4ed
2 files changed
+72
-4
tests/entrypoints/openai/chat_completion/test_serving_chat.py
+66
@@ -44,6 +44,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
44
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
45
from vllm.exceptions import QueueOverflowError, VLLMValidationError
46
from vllm.inputs import TokensPrompt
47
+from vllm.logprobs import Logprob
48
from vllm.multimodal.inputs import PlaceholderRange
49
from vllm.outputs import CompletionOutput, RequestOutput
50
from vllm.parser import HarmonyParser
@@ -845,6 +846,71 @@ async def test_streaming_reasoning_usage_counts_across_deltas():
846
}
847
848
849
+@pytest.mark.asyncio
850
+@pytest.mark.parametrize(
851
+ ("include_reasoning", "expected_tokens"),
852
+ [(True, ["<tool_call>", "answer"]), (False, [])],
853
+)
854
+async def test_streaming_logprobs_kept_when_parser_suppresses_delta(
855
+ include_reasoning: bool, expected_tokens: list[str]
856
+):
857
+ """Logprobs of tokens the parser swallows (e.g. <tool_call>) must still
858
+ be streamed, unless per-token metadata is hidden with the reasoning."""
859
+ serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
860
+ serving.model_config = None
861
+ serving.return_tokens_as_token_ids = False
862
+
863
+ parser = MagicMock()
864
+ parser.parse_delta.side_effect = [None, DeltaMessage(content="answer")]
865
+ parser.count_reasoning_tokens.return_value = 0
866
+ serving.parser_cls = MagicMock(return_value=parser)
867
+
868
+ def make_output(token_id: int, text: str, finish_reason: str | None):
869
+ request_output = _make_metrics_request_output(
870
+ metrics=None, token_ids=(token_id,)
871
+ )
872
+ completion = request_output.outputs[0]
873
+ completion.text = text
874
+ completion.finish_reason = finish_reason
875
+ completion.logprobs = [
876
+ {token_id: Logprob(logprob=-0.5, rank=1, decoded_token=text)}
877
+ ]
878
+ return request_output
879
+
880
+ request = ChatCompletionRequest(
881
+ model="test-model",
882
+ messages=[{"role": "user", "content": "Test prompt"}],
883
+ max_tokens=10,
884
+ stream=True,
885
+ logprobs=True,
886
+ include_reasoning=include_reasoning,
887
+ )
888
+ chunks: list[dict[str, Any]] = []
889
+ async for line in serving.chat_completion_stream_generator(
890
+ request,
891
+ _stream_request_outputs(
892
+ make_output(10, "<tool_call>", None), make_output(11, "answer", "stop")
893
+ ),
894
+ "chatcmpl-test-id",
895
+ "test-model",
896
+ conversation=[{"role": "user", "content": "Test"}],
897
+ tokenizer=MagicMock(),
898
+ request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
899
+ ):
900
+ payload = line.removeprefix("data: ").strip()
901
+ if payload != "[DONE]":
902
+ chunks.append(json.loads(payload))
903
+
904
+ streamed_tokens = [
905
+ entry["token"]
906
+ for chunk in chunks
907
+ for choice in chunk["choices"]
908
+ if choice.get("logprobs")
909
+ for entry in choice["logprobs"]["content"]
910
+ ]
911
+ assert streamed_tokens == expected_tokens
912
+
913
+
914
@dataclass
915
class MockEngine:
916
model_config: MockModelConfig = field(default_factory=MockModelConfig)
vllm/entrypoints/openai/chat_completion/serving.py
+6
-4
@@ -693,11 +693,13 @@ class OpenAIServingChat(GenerateBaseServing):
693
logprobs = None
694
695
if delta_message is None:
696
- # NOTE: If return_token_ids is enabled, we still need to
697
- # send a chunk with token_ids even if delta_message is None
696
+ # NOTE: If return_token_ids or logprobs are enabled, we
697
+ # still need to send a chunk even if delta_message is None
698
# to ensure all tokens are included in the response
699
- if output.finish_reason is None and (
700
- not request.return_token_ids or hide_stream_metadata
699
+ if (
700
+ output.finish_reason is None
701
+ and logprobs is None
702
+ and (not request.return_token_ids or hide_stream_metadata)
703
):
704
continue
705
delta_message = DeltaMessage()