@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c4bca740e8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix] fix missing last itl in openai completions benchmark (#5926)

mcalman committed Jun 28, 2024 at 22:34 UTC c4bca740e8498987184466d2f85ed43f1e1feb80
1 file changed +5 -6
benchmarks/backend_request_func.py
+5 -6
@@ -265,6 +265,9 @@ async def async_request_openai_completions(
265 else:
266 data = json.loads(chunk)
267
268 + # NOTE: Some completion API might have a last
269 + # usage summary response without a token so we
270 + # want to check a token was generated
271 if data["choices"][0]["text"]:
272 timestamp = time.perf_counter()
273 # First token
@@ -273,12 +276,8 @@ async def async_request_openai_completions(
276 output.ttft = ttft
277
278 # Decoding phase
276 - # NOTE: Some completion API might have a last
277 - # usage summary response without a token so we
278 - # do not want to include as inter-token-latency
279 - elif data.get("usage", None) is None:
280 - output.itl.append(timestamp -
281 - most_recent_timestamp)
279 + output.itl.append(timestamp -
280 + most_recent_timestamp)
281
282 most_recent_timestamp = timestamp
283 generated_text += data["choices"][0]["text"]