A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Benchmark] Add new H100 machine (#10547)
Simon Mo committed
Nov 21, 2024 at 18:27 UTC
aed074860a46536faf77bacd76d02efccbaf4a5d
2 files changed
+31
-21
.buildkite/nightly-benchmarks/benchmark-pipeline.yaml
+21
-18
@@ -13,6 +13,7 @@ steps:
13
- wait
14
15
- label: "A100"
16
+ # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing"
17
agents:
18
queue: A100
19
plugins:
@@ -45,6 +46,7 @@ steps:
46
medium: Memory
47
48
- label: "H200"
49
+ # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing"
50
agents:
51
queue: H200
52
plugins:
@@ -63,21 +65,22 @@ steps:
65
- VLLM_USAGE_SOURCE
66
- HF_TOKEN
67
66
-
67
- # - label: "H100"
68
- # agents:
69
- # queue: H100
70
- # plugins:
71
- # - docker#v5.11.0:
72
- # image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
73
- # command:
74
- # - bash
75
- # - .buildkite/nightly-benchmarks/run-benchmarks-suite.sh
76
- # mount-buildkite-agent: true
77
- # propagate-environment: true
78
- # ipc: host
79
- # gpus: all
80
- # environment:
81
- # - VLLM_USAGE_SOURCE
82
- # - HF_TOKEN
83
-
68
+ - label: "H100"
69
+ # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing"
70
+ agents:
71
+ queue: H100
72
+ plugins:
73
+ - docker#v5.12.0:
74
+ image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
75
+ command:
76
+ - bash
77
+ - .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
78
+ mount-buildkite-agent: true
79
+ propagate-environment: true
80
+ ipc: host
81
+ gpus: all # see CUDA_VISIBLE_DEVICES for actual GPUs used
82
+ volumes:
83
+ - /data/benchmark-hf-cache:/root/.cache/huggingface
84
+ environment:
85
+ - VLLM_USAGE_SOURCE
86
+ - HF_TOKEN
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+10
-3
@@ -157,10 +157,17 @@ if __name__ == "__main__":
157
throughput_results,
158
serving_results)
159
160
- # Sort all dataframes by their respective "Test name" columns
160
for df in [latency_results, serving_results, throughput_results]:
162
- if not df.empty:
163
- df.sort_values(by="Test name", inplace=True)
161
+ if df.empty:
162
+ continue
163
+
164
+ # Sort all dataframes by their respective "Test name" columns
165
+ df.sort_values(by="Test name", inplace=True)
166
+
167
+ # The GPUs sometimes come in format of "GPUTYPE\nGPUTYPE\n...",
168
+ # we want to turn it into "8xGPUTYPE"
169
+ df["GPU"] = df["GPU"].apply(
170
+ lambda x: f"{len(x.split('\n'))}x{x.split('\n')[0]}")
171
172
# get markdown tables
173
latency_md_table = tabulate(latency_results,