@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 52b7fcb35a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

Benchmark: add H100 suite (#6047)

Simon Mo committed Jul 11, 2024 at 09:17 UTC 52b7fcb35a6f8b57429431e929884c05d8266023
2 files changed +40 -23
.buildkite/nightly-benchmarks/benchmark-pipeline.yaml
+17 -18
@@ -11,7 +11,7 @@ steps:
11 - sh
12 - .buildkite/nightly-benchmarks/scripts/wait-for-image.sh
13 - wait
14 - - label: "A100 Benchmark"
14 + - label: "A100"
15 agents:
16 queue: A100
17 plugins:
@@ -42,21 +42,20 @@ steps:
42 - name: devshm
43 emptyDir:
44 medium: Memory
45 - # - label: "H100: NVIDIA SMI"
46 - # agents:
47 - # queue: H100
48 - # plugins:
49 - # - docker#v5.11.0:
50 - # image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
51 - # command:
52 - # - bash
53 - # - .buildkite/nightly-benchmarks/run-benchmarks-suite.sh
54 - # mount-buildkite-agent: true
55 - # propagate-environment: true
56 - # propagate-uid-gid: false
57 - # ipc: host
58 - # gpus: all
59 - # environment:
60 - # - VLLM_USAGE_SOURCE
61 - # - HF_TOKEN
45 + - label: "H100"
46 + agents:
47 + queue: H100
48 + plugins:
49 + - docker#v5.11.0:
50 + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
51 + command:
52 + - bash
53 + - .buildkite/nightly-benchmarks/run-benchmarks-suite.sh
54 + mount-buildkite-agent: true
55 + propagate-environment: true
56 + ipc: host
57 + gpus: all
58 + environment:
59 + - VLLM_USAGE_SOURCE
60 + - HF_TOKEN
61
.buildkite/nightly-benchmarks/run-benchmarks-suite.sh
+23 -5
@@ -54,7 +54,7 @@ wait_for_server() {
54 # wait for vllm server to start
55 # return 1 if vllm server crashes
56 timeout 1200 bash -c '
57 - until curl localhost:8000/v1/completions; do
57 + until curl -X POST localhost:8000/v1/completions; do
58 sleep 1
59 done' && return 0 || return 1
60 }
@@ -73,8 +73,17 @@ kill_gpu_processes() {
73 echo "All GPU processes have been killed."
74 fi
75
76 + # Sometimes kill with pid doesn't work properly, we can also kill all process running python or python3
77 + # since we are in container anyway
78 + pkill -9 -f python
79 + pkill -9 -f python3
80 +
81 # waiting for GPU processes to be fully killed
77 - sleep 10
82 + # loop while nvidia-smi returns any processes
83 + while [ -n "$(nvidia-smi --query-compute-apps=pid --format=csv,noheader)" ]; do
84 + sleep 1
85 + echo "Waiting for GPU processes to be killed"
86 + done
87
88 # remove vllm config file
89 rm -rf ~/.config/vllm
@@ -90,12 +99,19 @@ upload_to_buildkite() {
99 # upload the benchmarking results to buildkite
100
101 # if the agent binary is not found, skip uploading the results, exit 0
93 - if [ ! -f /workspace/buildkite-agent ]; then
102 + # Check if buildkite-agent is available in the PATH or at /workspace/buildkite-agent
103 + if command -v buildkite-agent >/dev/null 2>&1; then
104 + BUILDKITE_AGENT_COMMAND="buildkite-agent"
105 + elif [ -f /workspace/buildkite-agent ]; then
106 + BUILDKITE_AGENT_COMMAND="/workspace/buildkite-agent"
107 + else
108 echo "buildkite-agent binary not found. Skip uploading the results."
109 return 0
110 fi
97 - /workspace/buildkite-agent annotate --style "info" --context "benchmark-results" < $RESULTS_FOLDER/benchmark_results.md
98 - /workspace/buildkite-agent artifact upload "$RESULTS_FOLDER/*"
111 +
112 + # Use the determined command to annotate and upload artifacts
113 + $BUILDKITE_AGENT_COMMAND annotate --style "info" --context "$BUILDKITE_LABEL-benchmark-results" < $RESULTS_FOLDER/benchmark_results.md
114 + $BUILDKITE_AGENT_COMMAND artifact upload "$RESULTS_FOLDER/*"
115 }
116
117 run_latency_tests() {
@@ -269,6 +285,7 @@ run_serving_tests() {
285 echo "Running test case $test_name"
286 echo "Server command: $server_command"
287 eval "$server_command" &
288 + server_pid=$!
289
290 # wait until the server is alive
291 wait_for_server
@@ -318,6 +335,7 @@ run_serving_tests() {
335 done
336
337 # clean up
338 + kill -9 $server_pid
339 kill_gpu_processes
340 done
341 }