A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Bugfix] Add error handling when server cannot respond any valid tokens (#5895)
DearPlanet committed
Nov 8, 2024 at 12:58 UTC
ad39bd640cdaaf2963cd07a7cc912c1dde516ed0
1 file changed
+10
-3
benchmarks/backend_request_func.py
+10
-3
@@ -256,6 +256,7 @@ async def async_request_openai_completions(
256
async with session.post(url=api_url, json=payload,
257
headers=headers) as response:
258
if response.status == 200:
259
+ first_valid_chunk_received = False
260
async for chunk_bytes in response.content:
261
chunk_bytes = chunk_bytes.strip()
262
if not chunk_bytes:
@@ -274,7 +275,8 @@ async def async_request_openai_completions(
275
if data["choices"][0]["text"]:
276
timestamp = time.perf_counter()
277
# First token
277
- if ttft == 0.0:
278
+ if not first_valid_chunk_received:
279
+ first_chunk_received = True
280
ttft = time.perf_counter() - st
281
output.ttft = ttft
282
@@ -285,9 +287,14 @@ async def async_request_openai_completions(
287
288
most_recent_timestamp = timestamp
289
generated_text += data["choices"][0]["text"]
288
-
290
+ if first_chunk_received:
291
+ output.success = True
292
+ else:
293
+ output.success = False
294
+ output.error = (
295
+ "Never received a valid chunk to calculate TTFT."
296
+ "This response will be marked as failed!")
297
output.generated_text = generated_text
290
- output.success = True
298
output.latency = latency
299
else:
300
output.error = response.reason or ""