A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] fix missing last itl in openai completions benchmark (#5926)
mcalman committed
Jun 28, 2024 at 22:34 UTC
c4bca740e8498987184466d2f85ed43f1e1feb80
1 file changed
+5
-6
benchmarks/backend_request_func.py
+5
-6
@@ -265,6 +265,9 @@ async def async_request_openai_completions(
265
else:
266
data = json.loads(chunk)
267
268
+ # NOTE: Some completion API might have a last
269
+ # usage summary response without a token so we
270
+ # want to check a token was generated
271
if data["choices"][0]["text"]:
272
timestamp = time.perf_counter()
273
# First token
@@ -273,12 +276,8 @@ async def async_request_openai_completions(
276
output.ttft = ttft
277
278
# Decoding phase
276
- # NOTE: Some completion API might have a last
277
- # usage summary response without a token so we
278
- # do not want to include as inter-token-latency
279
- elif data.get("usage", None) is None:
280
- output.itl.append(timestamp -
281
- most_recent_timestamp)
279
+ output.itl.append(timestamp -
280
+ most_recent_timestamp)
281
282
most_recent_timestamp = timestamp
283
generated_text += data["choices"][0]["text"]