A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[perf bench] H200 development (#9768)
Signed-off-by: simon-mo <simon.mo@hey.com>
Simon Mo committed
Nov 20, 2024 at 11:06 UTC
5f1d6af2b619b07b2af3151d6aa59f9adc17e1eb
3 files changed
+32
-7
.buildkite/nightly-benchmarks/benchmark-pipeline.yaml
+23
@@ -9,7 +9,9 @@ steps:
9
- image: badouralix/curl-jq
10
command:
11
- sh .buildkite/nightly-benchmarks/scripts/wait-for-image.sh
12
+
13
- wait
14
+
15
- label: "A100"
16
agents:
17
queue: A100
@@ -41,6 +43,27 @@ steps:
43
- name: devshm
44
emptyDir:
45
medium: Memory
46
+
47
+ - label: "H200"
48
+ agents:
49
+ queue: H200
50
+ plugins:
51
+ - docker#v5.12.0:
52
+ image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
53
+ command:
54
+ - bash
55
+ - .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
56
+ mount-buildkite-agent: true
57
+ propagate-environment: true
58
+ ipc: host
59
+ gpus: 4,5,6,7
60
+ volumes:
61
+ - /data/benchmark-hf-cache:/root/.cache/huggingface
62
+ environment:
63
+ - VLLM_USAGE_SOURCE
64
+ - HF_TOKEN
65
+
66
+
67
# - label: "H100"
68
# agents:
69
# queue: H100
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+5
@@ -157,6 +157,11 @@ if __name__ == "__main__":
157
throughput_results,
158
serving_results)
159
160
+ # Sort all dataframes by their respective "Test name" columns
161
+ for df in [latency_results, serving_results, throughput_results]:
162
+ if not df.empty:
163
+ df.sort_values(by="Test name", inplace=True)
164
+
165
# get markdown tables
166
latency_md_table = tabulate(latency_results,
167
headers='keys',
.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
+4
-7
@@ -6,6 +6,7 @@
6
7
# Do not set -e, as the mixtral 8x22B model tends to crash occasionally
8
# and we still want to see other benchmarking results even when mixtral crashes.
9
+set -x
10
set -o pipefail
11
12
check_gpus() {
@@ -85,11 +86,7 @@ kill_gpu_processes() {
86
87
ps -aux
88
lsof -t -i:8000 | xargs -r kill -9
88
- pkill -f pt_main_thread
89
- # this line doesn't work now
90
- # ps aux | grep python | grep openai | awk '{print $2}' | xargs -r kill -9
91
- pkill -f python3
92
- pkill -f /usr/bin/python3
89
+ pgrep python3 | xargs -r kill -9
90
91
92
# wait until GPU memory usage smaller than 1GB
@@ -289,7 +286,7 @@ run_serving_tests() {
286
# run the server
287
echo "Running test case $test_name"
288
echo "Server command: $server_command"
292
- eval "$server_command" &
289
+ bash -c "$server_command" &
290
server_pid=$!
291
292
# wait until the server is alive
@@ -322,7 +319,7 @@ run_serving_tests() {
319
echo "Running test case $test_name with qps $qps"
320
echo "Client command: $client_command"
321
325
- eval "$client_command"
322
+ bash -c "$client_command"
323
324
# record the benchmarking commands
325
jq_output=$(jq -n \