A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
Benchmark: add H100 suite (#6047)
Simon Mo committed
Jul 11, 2024 at 09:17 UTC
52b7fcb35a6f8b57429431e929884c05d8266023
2 files changed
+40
-23
.buildkite/nightly-benchmarks/benchmark-pipeline.yaml
+17
-18
@@ -11,7 +11,7 @@ steps:
11
- sh
12
- .buildkite/nightly-benchmarks/scripts/wait-for-image.sh
13
- wait
14
- - label: "A100 Benchmark"
14
+ - label: "A100"
15
agents:
16
queue: A100
17
plugins:
@@ -42,21 +42,20 @@ steps:
42
- name: devshm
43
emptyDir:
44
medium: Memory
45
- # - label: "H100: NVIDIA SMI"
46
- # agents:
47
- # queue: H100
48
- # plugins:
49
- # - docker#v5.11.0:
50
- # image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
51
- # command:
52
- # - bash
53
- # - .buildkite/nightly-benchmarks/run-benchmarks-suite.sh
54
- # mount-buildkite-agent: true
55
- # propagate-environment: true
56
- # propagate-uid-gid: false
57
- # ipc: host
58
- # gpus: all
59
- # environment:
60
- # - VLLM_USAGE_SOURCE
61
- # - HF_TOKEN
45
+ - label: "H100"
46
+ agents:
47
+ queue: H100
48
+ plugins:
49
+ - docker#v5.11.0:
50
+ image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
51
+ command:
52
+ - bash
53
+ - .buildkite/nightly-benchmarks/run-benchmarks-suite.sh
54
+ mount-buildkite-agent: true
55
+ propagate-environment: true
56
+ ipc: host
57
+ gpus: all
58
+ environment:
59
+ - VLLM_USAGE_SOURCE
60
+ - HF_TOKEN
61
.buildkite/nightly-benchmarks/run-benchmarks-suite.sh
+23
-5
@@ -54,7 +54,7 @@ wait_for_server() {
54
# wait for vllm server to start
55
# return 1 if vllm server crashes
56
timeout 1200 bash -c '
57
- until curl localhost:8000/v1/completions; do
57
+ until curl -X POST localhost:8000/v1/completions; do
58
sleep 1
59
done' && return 0 || return 1
60
}
@@ -73,8 +73,17 @@ kill_gpu_processes() {
73
echo "All GPU processes have been killed."
74
fi
75
76
+ # Sometimes kill with pid doesn't work properly, we can also kill all process running python or python3
77
+ # since we are in container anyway
78
+ pkill -9 -f python
79
+ pkill -9 -f python3
80
+
81
# waiting for GPU processes to be fully killed
77
- sleep 10
82
+ # loop while nvidia-smi returns any processes
83
+ while [ -n "$(nvidia-smi --query-compute-apps=pid --format=csv,noheader)" ]; do
84
+ sleep 1
85
+ echo "Waiting for GPU processes to be killed"
86
+ done
87
88
# remove vllm config file
89
rm -rf ~/.config/vllm
@@ -90,12 +99,19 @@ upload_to_buildkite() {
99
# upload the benchmarking results to buildkite
100
101
# if the agent binary is not found, skip uploading the results, exit 0
93
- if [ ! -f /workspace/buildkite-agent ]; then
102
+ # Check if buildkite-agent is available in the PATH or at /workspace/buildkite-agent
103
+ if command -v buildkite-agent >/dev/null 2>&1; then
104
+ BUILDKITE_AGENT_COMMAND="buildkite-agent"
105
+ elif [ -f /workspace/buildkite-agent ]; then
106
+ BUILDKITE_AGENT_COMMAND="/workspace/buildkite-agent"
107
+ else
108
echo "buildkite-agent binary not found. Skip uploading the results."
109
return 0
110
fi
97
- /workspace/buildkite-agent annotate --style "info" --context "benchmark-results" < $RESULTS_FOLDER/benchmark_results.md
98
- /workspace/buildkite-agent artifact upload "$RESULTS_FOLDER/*"
111
+
112
+ # Use the determined command to annotate and upload artifacts
113
+ $BUILDKITE_AGENT_COMMAND annotate --style "info" --context "$BUILDKITE_LABEL-benchmark-results" < $RESULTS_FOLDER/benchmark_results.md
114
+ $BUILDKITE_AGENT_COMMAND artifact upload "$RESULTS_FOLDER/*"
115
}
116
117
run_latency_tests() {
@@ -269,6 +285,7 @@ run_serving_tests() {
285
echo "Running test case $test_name"
286
echo "Server command: $server_command"
287
eval "$server_command" &
288
+ server_pid=$!
289
290
# wait until the server is alive
291
wait_for_server
@@ -318,6 +335,7 @@ run_serving_tests() {
335
done
336
337
# clean up
338
+ kill -9 $server_pid
339
kill_gpu_processes
340
done
341
}