[Frontend] Omit absent fields from /inference/v1/generate stream chunks (#57264)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
aoshen02 committed
Sep 17, 2026 at 13:58 UTC
8538017f4b1d01b543345f5dbe1993cb66c80fa9
3 files changed
+67
-7
rust/src/server/src/routes/inference/generate.rs
+30
@@ -679,6 +679,36 @@ mod tests {
679
assert!(response.mm_placeholders.is_none());
680
}
681
682
+ #[test]
683
+ fn stream_chunk_omits_absent_fields() {
684
+ // Contract shared with the Python frontend: absent optional fields
685
+ // are omitted from stream chunks, not serialized as null.
686
+ let chunk = GenerateStreamResponse {
687
+ request_id: "raw-1".to_string(),
688
+ choices: vec![GenerateResponseStreamChoice {
689
+ index: 0,
690
+ logprobs: None,
691
+ finish_reason: None,
692
+ token_ids: vec![1],
693
+ }],
694
+ usage: None,
695
+ prompt_token_ids: None,
696
+ mm_placeholders: None,
697
+ };
698
+
699
+ let json = serde_json::to_value(&chunk).expect("serialize chunk");
700
+
701
+ for key in ["usage", "prompt_token_ids", "mm_placeholders"] {
702
+ assert!(json.get(key).is_none(), "{key} should be omitted");
703
+ }
704
+ for key in ["logprobs", "finish_reason"] {
705
+ assert!(
706
+ json["choices"][0].get(key).is_none(),
707
+ "{key} should be omitted"
708
+ );
709
+ }
710
+ }
711
+
712
#[test]
713
fn extract_mm_placeholders_returns_none_without_features() {
714
assert!(extract_mm_placeholders(None).is_none());
tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py
+34
@@ -569,6 +569,40 @@ async def test_stream_omits_prompt_metadata_by_default():
569
assert "mm_placeholders" not in chunk
570
571
572
+@pytest.mark.asyncio
573
+async def test_stream_chunks_omit_absent_fields():
574
+ """Absent optional fields are omitted from stream chunks, like Rust."""
575
+ engine = _mock_engine()
576
+
577
+ async def mock_generate(*args, **kwargs):
578
+ yield _make_request_output("req-1", token_ids=[10])
579
+ yield _make_request_output(
580
+ "req-1", token_ids=[20], finish_reason="stop", finished=True
581
+ )
582
+
583
+ engine.generate = MagicMock(side_effect=mock_generate)
584
+ serving = _build_serving_tokens(engine)
585
+ request = GenerateRequest(
586
+ token_ids=[1, 2, 3],
587
+ sampling_params=SamplingParams(max_tokens=2),
588
+ model=MODEL_NAME,
589
+ stream=True,
590
+ )
591
+
592
+ response = await serving.serve_tokens(request)
593
+ parsed = _parse_sse_chunks([chunk async for chunk in response])
594
+ data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")]
595
+
596
+ assert len(data_chunks) == 2
597
+ for chunk in data_chunks:
598
+ assert "usage" not in chunk
599
+ assert "prompt_token_ids" not in chunk
600
+ assert "mm_placeholders" not in chunk
601
+ assert "logprobs" not in chunk["choices"][0]
602
+ assert "finish_reason" not in data_chunks[0]["choices"][0]
603
+ assert data_chunks[1]["choices"][0]["finish_reason"] == "stop"
604
+
605
+
606
@pytest.mark.asyncio
607
async def test_stream_text_only_prompt_omits_mm_placeholders_key():
608
engine = _mock_engine()
vllm/entrypoints/scale_out/token_in_token_out/serving.py
+3
-7
@@ -486,13 +486,9 @@ class ServingTokens(GenerateBaseServing):
486
total_tokens=(num_prompt_tokens + num_generated_tokens[i]),
487
)
488
489
- # Omit absent prompt metadata (Rust skips None fields).
490
- exclude = {
491
- name
492
- for name in ("prompt_token_ids", "mm_placeholders")
493
- if getattr(chunk, name) is None
494
- }
495
- yield f"data: {chunk.model_dump_json(exclude=exclude)}\n\n"
489
+ # Omit absent fields, like the Rust frontend and the
490
+ # final usage chunk below.
491
+ yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n"
492
493
total_completion_tokens = sum(num_generated_tokens)
494
final_usage_info = UsageInfo(