@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aed074860a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Benchmark] Add new H100 machine (#10547)

Simon Mo committed Nov 21, 2024 at 18:27 UTC aed074860a46536faf77bacd76d02efccbaf4a5d
2 files changed +31 -21
.buildkite/nightly-benchmarks/benchmark-pipeline.yaml
+21 -18
@@ -13,6 +13,7 @@ steps:
13 - wait
14
15 - label: "A100"
16 + # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing"
17 agents:
18 queue: A100
19 plugins:
@@ -45,6 +46,7 @@ steps:
46 medium: Memory
47
48 - label: "H200"
49 + # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing"
50 agents:
51 queue: H200
52 plugins:
@@ -63,21 +65,22 @@ steps:
65 - VLLM_USAGE_SOURCE
66 - HF_TOKEN
67
66 -
67 - # - label: "H100"
68 - # agents:
69 - # queue: H100
70 - # plugins:
71 - # - docker#v5.11.0:
72 - # image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
73 - # command:
74 - # - bash
75 - # - .buildkite/nightly-benchmarks/run-benchmarks-suite.sh
76 - # mount-buildkite-agent: true
77 - # propagate-environment: true
78 - # ipc: host
79 - # gpus: all
80 - # environment:
81 - # - VLLM_USAGE_SOURCE
82 - # - HF_TOKEN
83 -
68 + - label: "H100"
69 + # skip: "use this flag to conditionally skip the benchmark step, useful for PR testing"
70 + agents:
71 + queue: H100
72 + plugins:
73 + - docker#v5.12.0:
74 + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
75 + command:
76 + - bash
77 + - .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
78 + mount-buildkite-agent: true
79 + propagate-environment: true
80 + ipc: host
81 + gpus: all # see CUDA_VISIBLE_DEVICES for actual GPUs used
82 + volumes:
83 + - /data/benchmark-hf-cache:/root/.cache/huggingface
84 + environment:
85 + - VLLM_USAGE_SOURCE
86 + - HF_TOKEN
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+10 -3
@@ -157,10 +157,17 @@ if __name__ == "__main__":
157 throughput_results,
158 serving_results)
159
160 - # Sort all dataframes by their respective "Test name" columns
160 for df in [latency_results, serving_results, throughput_results]:
162 - if not df.empty:
163 - df.sort_values(by="Test name", inplace=True)
161 + if df.empty:
162 + continue
163 +
164 + # Sort all dataframes by their respective "Test name" columns
165 + df.sort_values(by="Test name", inplace=True)
166 +
167 + # The GPUs sometimes come in format of "GPUTYPE\nGPUTYPE\n...",
168 + # we want to turn it into "8xGPUTYPE"
169 + df["GPU"] = df["GPU"].apply(
170 + lambda x: f"{len(x.split('\n'))}x{x.split('\n')[0]}")
171
172 # get markdown tables
173 latency_md_table = tabulate(latency_results,