@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ccb58b23e6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Misc] Fix Benchmark TTFT Calculation for Chat Completions (#3768)

Roger Wang committed Apr 1, 2024 at 15:24 UTC ccb58b23e698ed0bd3f4a850366211622463f109
1 file changed +3 -3
benchmarks/backend_request_func.py
+3 -3
@@ -334,7 +334,8 @@ async def async_request_openai_chat_completions(
334 timestamp = time.perf_counter()
335 data = json.loads(chunk)
336
337 - if "content" in data["choices"][0]["delta"]:
337 + delta = data["choices"][0]["delta"]
338 + if delta.get("content", None):
339 # First token
340 if ttft == 0:
341 ttft = time.perf_counter() - st
@@ -345,8 +346,7 @@ async def async_request_openai_chat_completions(
346 output.itl.append(timestamp -
347 most_recent_timestamp)
348
348 - generated_text += data["choices"][0]["delta"][
349 - "content"]
349 + generated_text += delta["content"]
350
351 most_recent_timestamp = timestamp
352