@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ad39bd640c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix] Add error handling when server cannot respond any valid tokens (#5895)

DearPlanet committed Nov 8, 2024 at 12:58 UTC ad39bd640cdaaf2963cd07a7cc912c1dde516ed0
1 file changed +10 -3
benchmarks/backend_request_func.py
+10 -3
@@ -256,6 +256,7 @@ async def async_request_openai_completions(
256 async with session.post(url=api_url, json=payload,
257 headers=headers) as response:
258 if response.status == 200:
259 + first_valid_chunk_received = False
260 async for chunk_bytes in response.content:
261 chunk_bytes = chunk_bytes.strip()
262 if not chunk_bytes:
@@ -274,7 +275,8 @@ async def async_request_openai_completions(
275 if data["choices"][0]["text"]:
276 timestamp = time.perf_counter()
277 # First token
277 - if ttft == 0.0:
278 + if not first_valid_chunk_received:
279 + first_chunk_received = True
280 ttft = time.perf_counter() - st
281 output.ttft = ttft
282
@@ -285,9 +287,14 @@ async def async_request_openai_completions(
287
288 most_recent_timestamp = timestamp
289 generated_text += data["choices"][0]["text"]
288 -
290 + if first_chunk_received:
291 + output.success = True
292 + else:
293 + output.success = False
294 + output.error = (
295 + "Never received a valid chunk to calculate TTFT."
296 + "This response will be marked as failed!")
297 output.generated_text = generated_text
290 - output.success = True
298 output.latency = latency
299 else:
300 output.error = response.reason or ""