@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5f1d6af2b6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[perf bench] H200 development (#9768)

Signed-off-by: simon-mo <simon.mo@hey.com>

Simon Mo committed Nov 20, 2024 at 11:06 UTC 5f1d6af2b619b07b2af3151d6aa59f9adc17e1eb
3 files changed +32 -7
.buildkite/nightly-benchmarks/benchmark-pipeline.yaml
+23
@@ -9,7 +9,9 @@ steps:
9 - image: badouralix/curl-jq
10 command:
11 - sh .buildkite/nightly-benchmarks/scripts/wait-for-image.sh
12 +
13 - wait
14 +
15 - label: "A100"
16 agents:
17 queue: A100
@@ -41,6 +43,27 @@ steps:
43 - name: devshm
44 emptyDir:
45 medium: Memory
46 +
47 + - label: "H200"
48 + agents:
49 + queue: H200
50 + plugins:
51 + - docker#v5.12.0:
52 + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
53 + command:
54 + - bash
55 + - .buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
56 + mount-buildkite-agent: true
57 + propagate-environment: true
58 + ipc: host
59 + gpus: 4,5,6,7
60 + volumes:
61 + - /data/benchmark-hf-cache:/root/.cache/huggingface
62 + environment:
63 + - VLLM_USAGE_SOURCE
64 + - HF_TOKEN
65 +
66 +
67 # - label: "H100"
68 # agents:
69 # queue: H100
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+5
@@ -157,6 +157,11 @@ if __name__ == "__main__":
157 throughput_results,
158 serving_results)
159
160 + # Sort all dataframes by their respective "Test name" columns
161 + for df in [latency_results, serving_results, throughput_results]:
162 + if not df.empty:
163 + df.sort_values(by="Test name", inplace=True)
164 +
165 # get markdown tables
166 latency_md_table = tabulate(latency_results,
167 headers='keys',
.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
+4 -7
@@ -6,6 +6,7 @@
6
7 # Do not set -e, as the mixtral 8x22B model tends to crash occasionally
8 # and we still want to see other benchmarking results even when mixtral crashes.
9 +set -x
10 set -o pipefail
11
12 check_gpus() {
@@ -85,11 +86,7 @@ kill_gpu_processes() {
86
87 ps -aux
88 lsof -t -i:8000 | xargs -r kill -9
88 - pkill -f pt_main_thread
89 - # this line doesn't work now
90 - # ps aux | grep python | grep openai | awk '{print $2}' | xargs -r kill -9
91 - pkill -f python3
92 - pkill -f /usr/bin/python3
89 + pgrep python3 | xargs -r kill -9
90
91
92 # wait until GPU memory usage smaller than 1GB
@@ -289,7 +286,7 @@ run_serving_tests() {
286 # run the server
287 echo "Running test case $test_name"
288 echo "Server command: $server_command"
292 - eval "$server_command" &
289 + bash -c "$server_command" &
290 server_pid=$!
291
292 # wait until the server is alive
@@ -322,7 +319,7 @@ run_serving_tests() {
319 echo "Running test case $test_name with qps $qps"
320 echo "Client command: $client_command"
321
325 - eval "$client_command"
322 + bash -c "$client_command"
323
324 # record the benchmarking commands
325 jq_output=$(jq -n \