A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Fix ITL recording in serving benchmark (#7372)
Roger Wang committed
Aug 9, 2024 at 10:00 UTC
70d268a39947a8ea950f871f9345aad21f09715e
1 file changed
+3
-2
benchmarks/backend_request_func.py
+3
-2
@@ -276,8 +276,9 @@ async def async_request_openai_completions(
276
output.ttft = ttft
277
278
# Decoding phase
279
- output.itl.append(timestamp -
280
- most_recent_timestamp)
279
+ else:
280
+ output.itl.append(timestamp -
281
+ most_recent_timestamp)
282
283
most_recent_timestamp = timestamp
284
generated_text += data["choices"][0]["text"]