A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Bugfix] Fix E2EL mean and median stats (#9984)
Signed-off-by: daitran2k1 <tranquangdai7a@gmail.com>
Tran Quang Dai committed
Nov 4, 2024 at 16:37 UTC
ea4adeddc12412ad0854f93882e214000e91ce05
1 file changed
+2
-2
benchmarks/benchmark_serving.py
+2
-2
@@ -406,9 +406,9 @@ def calculate_metrics(
406
median_itl_ms=np.median(itls or 0) * 1000,
407
percentiles_itl_ms=[(p, np.percentile(itls or 0, p) * 1000)
408
for p in selected_percentiles],
409
- mean_e2el_ms=np.median(e2els or 0) * 1000,
409
+ mean_e2el_ms=np.mean(e2els or 0) * 1000,
410
std_e2el_ms=np.std(e2els or 0) * 1000,
411
- median_e2el_ms=np.mean(e2els or 0) * 1000,
411
+ median_e2el_ms=np.median(e2els or 0) * 1000,
412
percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000)
413
for p in selected_percentiles],
414
)