[Bugfix] Reject tokenless chat and audio streams (#54708)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Taneem Ibrahim committed
Sep 1, 2026 at 10:24 UTC
d9eb4e344f91838ef718c0f2123c836a58e49f08
1 file changed
+24
-10
vllm/benchmarks/lib/endpoint_request_func.py
+24
-10
@@ -376,13 +376,13 @@ async def async_request_openai_chat_completions(
376
output.prompt_len = request_func_input.prompt_len
377
378
generated_text = ""
379
- ttft = 0.0
379
st = time.perf_counter()
380
output.start_time = st
381
most_recent_timestamp = st
382
try:
383
async with session.post(url=api_url, json=payload, headers=headers) as response:
384
if response.status == 200:
385
+ first_chunk_received = False
386
handler = StreamedResponseHandler()
387
async for chunk_bytes in response.content.iter_any():
388
chunk_bytes = chunk_bytes.strip()
@@ -406,9 +406,9 @@ async def async_request_openai_chat_completions(
406
if choices := data.get("choices"):
407
content = choices[0]["delta"].get("content")
408
# First token
409
- if ttft == 0.0:
410
- ttft = timestamp - st
411
- output.ttft = ttft
409
+ if not first_chunk_received:
410
+ first_chunk_received = True
411
+ output.ttft = timestamp - st
412
413
# Decoding phase
414
else:
@@ -423,7 +423,14 @@ async def async_request_openai_chat_completions(
423
most_recent_timestamp = timestamp
424
425
output.generated_text = generated_text
426
- output.success = True
426
+ if first_chunk_received:
427
+ output.success = True
428
+ else:
429
+ output.success = False
430
+ output.error = (
431
+ "Never received a valid chunk to calculate TTFT."
432
+ "This response will be marked as failed!"
433
+ )
434
output.latency = most_recent_timestamp - st
435
else:
436
output.error = response.reason or ""
@@ -494,7 +501,6 @@ async def async_request_openai_audio(
501
output.input_audio_duration = input_audio_duration
502
503
generated_text = ""
497
- ttft = 0.0
504
st = time.perf_counter()
505
output.start_time = st
506
most_recent_timestamp = st
@@ -503,6 +509,7 @@ async def async_request_openai_audio(
509
url=api_url, data=form, headers=headers
510
) as response:
511
if response.status == 200:
512
+ first_chunk_received = False
513
handler = StreamedResponseHandler()
514
515
async for chunk_bytes in response.content.iter_any():
@@ -522,9 +529,9 @@ async def async_request_openai_audio(
529
if choices := data.get("choices"):
530
content = choices[0]["delta"].get("content")
531
# First token
525
- if ttft == 0.0:
526
- ttft = timestamp - st
527
- output.ttft = ttft
532
+ if not first_chunk_received:
533
+ first_chunk_received = True
534
+ output.ttft = timestamp - st
535
536
# Decoding phase
537
else:
@@ -541,7 +548,14 @@ async def async_request_openai_audio(
548
most_recent_timestamp = timestamp
549
550
output.generated_text = generated_text
544
- output.success = True
551
+ if first_chunk_received:
552
+ output.success = True
553
+ else:
554
+ output.success = False
555
+ output.error = (
556
+ "Never received a valid chunk to calculate TTFT."
557
+ "This response will be marked as failed!"
558
+ )
559
output.latency = most_recent_timestamp - st
560
else:
561
output.error = response.reason or ""