A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Misc] Fix Benchmark TTFT Calculation for Chat Completions (#3768)
Roger Wang committed
Apr 1, 2024 at 15:24 UTC
ccb58b23e698ed0bd3f4a850366211622463f109
1 file changed
+3
-3
benchmarks/backend_request_func.py
+3
-3
@@ -334,7 +334,8 @@ async def async_request_openai_chat_completions(
334
timestamp = time.perf_counter()
335
data = json.loads(chunk)
336
337
- if "content" in data["choices"][0]["delta"]:
337
+ delta = data["choices"][0]["delta"]
338
+ if delta.get("content", None):
339
# First token
340
if ttft == 0:
341
ttft = time.perf_counter() - st
@@ -345,8 +346,7 @@ async def async_request_openai_chat_completions(
346
output.itl.append(timestamp -
347
most_recent_timestamp)
348
348
- generated_text += data["choices"][0]["delta"][
349
- "content"]
349
+ generated_text += delta["content"]
350
351
most_recent_timestamp = timestamp
352