Revert "[Frontend] Omit absent fields from /inference/v1/generate stream chunks" (#57302)
aoshen02 committed
Sep 17, 2026 at 14:31 UTC
b978df3299518ed7b02584f69d6daac55eab0b71
3 files changed
+7
-67
rust/src/server/src/routes/inference/generate.rs
-30
@@ -679,36 +679,6 @@ mod tests {
679
assert!(response.mm_placeholders.is_none());
680
}
681
682
- #[test]
683
- fn stream_chunk_omits_absent_fields() {
684
- // Contract shared with the Python frontend: absent optional fields
685
- // are omitted from stream chunks, not serialized as null.
686
- let chunk = GenerateStreamResponse {
687
- request_id: "raw-1".to_string(),
688
- choices: vec![GenerateResponseStreamChoice {
689
- index: 0,
690
- logprobs: None,
691
- finish_reason: None,
692
- token_ids: vec![1],
693
- }],
694
- usage: None,
695
- prompt_token_ids: None,
696
- mm_placeholders: None,
697
- };
698
-
699
- let json = serde_json::to_value(&chunk).expect("serialize chunk");
700
-
701
- for key in ["usage", "prompt_token_ids", "mm_placeholders"] {
702
- assert!(json.get(key).is_none(), "{key} should be omitted");
703
- }
704
- for key in ["logprobs", "finish_reason"] {
705
- assert!(
706
- json["choices"][0].get(key).is_none(),
707
- "{key} should be omitted"
708
- );
709
- }
710
- }
711
-
682
#[test]
683
fn extract_mm_placeholders_returns_none_without_features() {
684
assert!(extract_mm_placeholders(None).is_none());
tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py
-34
@@ -569,40 +569,6 @@ async def test_stream_omits_prompt_metadata_by_default():
569
assert "mm_placeholders" not in chunk
570
571
572
-@pytest.mark.asyncio
573
-async def test_stream_chunks_omit_absent_fields():
574
- """Absent optional fields are omitted from stream chunks, like Rust."""
575
- engine = _mock_engine()
576
-
577
- async def mock_generate(*args, **kwargs):
578
- yield _make_request_output("req-1", token_ids=[10])
579
- yield _make_request_output(
580
- "req-1", token_ids=[20], finish_reason="stop", finished=True
581
- )
582
-
583
- engine.generate = MagicMock(side_effect=mock_generate)
584
- serving = _build_serving_tokens(engine)
585
- request = GenerateRequest(
586
- token_ids=[1, 2, 3],
587
- sampling_params=SamplingParams(max_tokens=2),
588
- model=MODEL_NAME,
589
- stream=True,
590
- )
591
-
592
- response = await serving.serve_tokens(request)
593
- parsed = _parse_sse_chunks([chunk async for chunk in response])
594
- data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")]
595
-
596
- assert len(data_chunks) == 2
597
- for chunk in data_chunks:
598
- assert "usage" not in chunk
599
- assert "prompt_token_ids" not in chunk
600
- assert "mm_placeholders" not in chunk
601
- assert "logprobs" not in chunk["choices"][0]
602
- assert "finish_reason" not in data_chunks[0]["choices"][0]
603
- assert data_chunks[1]["choices"][0]["finish_reason"] == "stop"
604
-
605
-
572
@pytest.mark.asyncio
573
async def test_stream_text_only_prompt_omits_mm_placeholders_key():
574
engine = _mock_engine()
vllm/entrypoints/scale_out/token_in_token_out/serving.py
+7
-3
@@ -486,9 +486,13 @@ class ServingTokens(GenerateBaseServing):
486
total_tokens=(num_prompt_tokens + num_generated_tokens[i]),
487
)
488
489
- # Omit absent fields, like the Rust frontend and the
490
- # final usage chunk below.
491
- yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n"
489
+ # Omit absent prompt metadata (Rust skips None fields).
490
+ exclude = {
491
+ name
492
+ for name in ("prompt_token_ids", "mm_placeholders")
493
+ if getattr(chunk, name) is None
494
+ }
495
+ yield f"data: {chunk.model_dump_json(exclude=exclude)}\n\n"
496
497
total_completion_tokens = sum(num_generated_tokens)
498
final_usage_info = UsageInfo(