@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 114d7270ff
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[CI] Avoid naming different metrics with the same name in performance benchmark (#5615)

Kuntai Du committed Jun 17, 2024 at 21:37 UTC 114d7270ffc2e5a66e0974b0d6d913c7f990afa7
1 file changed +14 -7
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+14 -7
@@ -15,10 +15,10 @@ latency_column_mapping = {
15 "avg_latency": "Mean latency (ms)",
16 # "P10": "P10 (s)",
17 # "P25": "P25 (s)",
18 - "P50": "Median",
18 + "P50": "Median latency (ms)",
19 # "P75": "P75 (s)",
20 # "P90": "P90 (s)",
21 - "P99": "P99",
21 + "P99": "P99 latency (ms)",
22 }
23
24 # throughput tests and the keys that will be printed into markdown
@@ -43,15 +43,14 @@ serving_column_mapping = {
43 # "input_throughput": "Input Tput (tok/s)",
44 # "output_throughput": "Output Tput (tok/s)",
45 "mean_ttft_ms": "Mean TTFT (ms)",
46 - # do not say TTFT again to avoid the table getting too wide
47 - "median_ttft_ms": "Median",
48 - "p99_ttft_ms": "P99",
46 + "median_ttft_ms": "Median TTFT (ms)",
47 + "p99_ttft_ms": "P99 TTFT (ms)",
48 # "mean_tpot_ms": "Mean TPOT (ms)",
49 # "median_tpot_ms": "Median",
50 # "p99_tpot_ms": "P99",
51 "mean_itl_ms": "Mean ITL (ms)",
53 - "median_itl_ms": "Median",
54 - "p99_itl_ms": "P99",
52 + "median_itl_ms": "Median ITL (ms)",
53 + "p99_itl_ms": "P99 ITL (ms)",
54 }
55
56
@@ -183,3 +182,11 @@ if __name__ == "__main__":
182 serving_tests_markdown_table=serving_md_table,
183 benchmarking_results_in_json_string=processed_results_json)
184 f.write(results)
185 +
186 + # document benchmarking results in json
187 + with open(results_folder / "benchmark_results.json", "w") as f:
188 +
189 + results = latency_results.to_dict(
190 + orient='records') + throughput_results.to_dict(
191 + orient='records') + serving_results.to_dict(orient='records')
192 + f.write(json.dumps(results))