@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3be5b26a76
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[CI/Build] Add shell script linting using shellcheck (#7925)

Signed-off-by: Russell Bryant <rbryant@redhat.com>

Russell Bryant committed Nov 7, 2024 at 13:17 UTC 3be5b26a7651b57aeb2cbdfc6aee81152ba68da5
28 files changed +204 -129
.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh
+3 -3
@@ -41,6 +41,6 @@ while getopts "m:b:l:f:" OPT; do
41 done
42
43 lm_eval --model hf \
44 - --model_args pretrained=$MODEL,parallelize=True \
45 - --tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
46 - --batch_size $BATCH_SIZE
44 + --model_args "pretrained=$MODEL,parallelize=True" \
45 + --tasks gsm8k --num_fewshot "$FEWSHOT" --limit "$LIMIT" \
46 + --batch_size "$BATCH_SIZE"
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+3 -3
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
46 done
47
48 lm_eval --model vllm \
49 - --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,distributed_executor_backend="ray",trust_remote_code=true,max_model_len=4096 \
50 - --tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
51 - --batch_size $BATCH_SIZE
49 + --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,distributed_executor_backend=ray,trust_remote_code=true,max_model_len=4096" \
50 + --tasks gsm8k --num_fewshot "$FEWSHOT" --limit "$LIMIT" \
51 + --batch_size "$BATCH_SIZE"
.buildkite/lm-eval-harness/run-tests.sh
+1 -1
@@ -30,7 +30,7 @@ while getopts "c:t:" OPT; do
30 done
31
32 # Parse list of configs.
33 -IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < $CONFIG
33 +IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "$CONFIG"
34
35 for MODEL_CONFIG in "${MODEL_CONFIGS[@]}"
36 do
.buildkite/nightly-benchmarks/scripts/launch-server.sh
+25 -38
@@ -50,31 +50,30 @@ launch_trt_server() {
50 git clone https://github.com/triton-inference-server/tensorrtllm_backend.git
51 git lfs install
52 cd tensorrtllm_backend
53 - git checkout $trt_llm_version
54 - tensorrtllm_backend_dir=$(pwd)
53 + git checkout "$trt_llm_version"
54 git submodule update --init --recursive
55
56 # build trtllm engine
57 cd /tensorrtllm_backend
59 - cd ./tensorrt_llm/examples/${model_type}
58 + cd "./tensorrt_llm/examples/${model_type}"
59 python3 convert_checkpoint.py \
61 - --model_dir ${model_path} \
62 - --dtype ${model_dtype} \
63 - --tp_size ${model_tp_size} \
64 - --output_dir ${trt_model_path}
60 + --model_dir "${model_path}" \
61 + --dtype "${model_dtype}" \
62 + --tp_size "${model_tp_size}" \
63 + --output_dir "${trt_model_path}"
64 trtllm-build \
66 - --checkpoint_dir ${trt_model_path} \
65 + --checkpoint_dir "${trt_model_path}" \
66 --use_fused_mlp \
67 --reduce_fusion disable \
68 --workers 8 \
70 - --gpt_attention_plugin ${model_dtype} \
71 - --gemm_plugin ${model_dtype} \
72 - --tp_size ${model_tp_size} \
73 - --max_batch_size ${max_batch_size} \
74 - --max_input_len ${max_input_len} \
75 - --max_seq_len ${max_seq_len} \
76 - --max_num_tokens ${max_num_tokens} \
77 - --output_dir ${trt_engine_path}
69 + --gpt_attention_plugin "${model_dtype}" \
70 + --gemm_plugin "${model_dtype}" \
71 + --tp_size "${model_tp_size}" \
72 + --max_batch_size "${max_batch_size}" \
73 + --max_input_len "${max_input_len}" \
74 + --max_seq_len "${max_seq_len}" \
75 + --max_num_tokens "${max_num_tokens}" \
76 + --output_dir "${trt_engine_path}"
77
78 # handle triton protobuf files and launch triton server
79 cd /tensorrtllm_backend
@@ -82,15 +81,15 @@ launch_trt_server() {
81 cp -r all_models/inflight_batcher_llm/* triton_model_repo/
82 cd triton_model_repo
83 rm -rf ./tensorrt_llm/1/*
85 - cp -r ${trt_engine_path}/* ./tensorrt_llm/1
84 + cp -r "${trt_engine_path}"/* ./tensorrt_llm/1
85 python3 ../tools/fill_template.py -i tensorrt_llm/config.pbtxt triton_backend:tensorrtllm,engine_dir:/tensorrtllm_backend/triton_model_repo/tensorrt_llm/1,decoupled_mode:true,batching_strategy:inflight_fused_batching,batch_scheduler_policy:guaranteed_no_evict,exclude_input_in_output:true,triton_max_batch_size:2048,max_queue_delay_microseconds:0,max_beam_width:1,max_queue_size:2048,enable_kv_cache_reuse:false
87 - python3 ../tools/fill_template.py -i preprocessing/config.pbtxt triton_max_batch_size:2048,tokenizer_dir:$model_path,preprocessing_instance_count:5
88 - python3 ../tools/fill_template.py -i postprocessing/config.pbtxt triton_max_batch_size:2048,tokenizer_dir:$model_path,postprocessing_instance_count:5,skip_special_tokens:false
89 - python3 ../tools/fill_template.py -i ensemble/config.pbtxt triton_max_batch_size:$max_batch_size
90 - python3 ../tools/fill_template.py -i tensorrt_llm_bls/config.pbtxt triton_max_batch_size:$max_batch_size,decoupled_mode:true,accumulate_tokens:"False",bls_instance_count:1
86 + python3 ../tools/fill_template.py -i preprocessing/config.pbtxt "triton_max_batch_size:2048,tokenizer_dir:$model_path,preprocessing_instance_count:5"
87 + python3 ../tools/fill_template.py -i postprocessing/config.pbtxt "triton_max_batch_size:2048,tokenizer_dir:$model_path,postprocessing_instance_count:5,skip_special_tokens:false"
88 + python3 ../tools/fill_template.py -i ensemble/config.pbtxt triton_max_batch_size:"$max_batch_size"
89 + python3 ../tools/fill_template.py -i tensorrt_llm_bls/config.pbtxt "triton_max_batch_size:$max_batch_size,decoupled_mode:true,accumulate_tokens:False,bls_instance_count:1"
90 cd /tensorrtllm_backend
91 python3 scripts/launch_triton_server.py \
93 - --world_size=${model_tp_size} \
92 + --world_size="${model_tp_size}" \
93 --model_repo=/tensorrtllm_backend/triton_model_repo &
94
95 }
@@ -98,10 +97,7 @@ launch_trt_server() {
97 launch_tgi_server() {
98 model=$(echo "$common_params" | jq -r '.model')
99 tp=$(echo "$common_params" | jq -r '.tp')
101 - dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
102 - dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
100 port=$(echo "$common_params" | jq -r '.port')
104 - num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
101 server_args=$(json2args "$server_params")
102
103 if echo "$common_params" | jq -e 'has("fp8")' >/dev/null; then
@@ -129,10 +125,7 @@ launch_tgi_server() {
125 launch_lmdeploy_server() {
126 model=$(echo "$common_params" | jq -r '.model')
127 tp=$(echo "$common_params" | jq -r '.tp')
132 - dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
133 - dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
128 port=$(echo "$common_params" | jq -r '.port')
135 - num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
129 server_args=$(json2args "$server_params")
130
131 server_command="lmdeploy serve api_server $model \
@@ -149,10 +142,7 @@ launch_sglang_server() {
142
143 model=$(echo "$common_params" | jq -r '.model')
144 tp=$(echo "$common_params" | jq -r '.tp')
152 - dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
153 - dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
145 port=$(echo "$common_params" | jq -r '.port')
155 - num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
146 server_args=$(json2args "$server_params")
147
148 if echo "$common_params" | jq -e 'has("fp8")' >/dev/null; then
@@ -185,10 +175,7 @@ launch_vllm_server() {
175
176 model=$(echo "$common_params" | jq -r '.model')
177 tp=$(echo "$common_params" | jq -r '.tp')
188 - dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
189 - dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
178 port=$(echo "$common_params" | jq -r '.port')
191 - num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
179 server_args=$(json2args "$server_params")
180
181 if echo "$common_params" | jq -e 'has("fp8")' >/dev/null; then
@@ -217,19 +204,19 @@ launch_vllm_server() {
204
205 main() {
206
220 - if [[ $CURRENT_LLM_SERVING_ENGINE == "trt" ]]; then
207 + if [[ "$CURRENT_LLM_SERVING_ENGINE" == "trt" ]]; then
208 launch_trt_server
209 fi
210
224 - if [[ $CURRENT_LLM_SERVING_ENGINE == "tgi" ]]; then
211 + if [[ "$CURRENT_LLM_SERVING_ENGINE" == "tgi" ]]; then
212 launch_tgi_server
213 fi
214
228 - if [[ $CURRENT_LLM_SERVING_ENGINE == "lmdeploy" ]]; then
215 + if [[ "$CURRENT_LLM_SERVING_ENGINE" == "lmdeploy" ]]; then
216 launch_lmdeploy_server
217 fi
218
232 - if [[ $CURRENT_LLM_SERVING_ENGINE == "sglang" ]]; then
219 + if [[ "$CURRENT_LLM_SERVING_ENGINE" == "sglang" ]]; then
220 launch_sglang_server
221 fi
222
.buildkite/nightly-benchmarks/scripts/nightly-annotate.sh
+6 -6
@@ -16,10 +16,10 @@ main() {
16 fi
17
18 # initial annotation
19 - description="$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/nightly-descriptions.md"
19 + #description="$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/nightly-descriptions.md"
20
21 # download results
22 - cd $VLLM_SOURCE_CODE_LOC/benchmarks
22 + cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
23 mkdir -p results/
24 /workspace/buildkite-agent artifact download 'results/*nightly_results.json' results/
25 ls
@@ -30,15 +30,15 @@ main() {
30 /workspace/buildkite-agent artifact upload "results.zip"
31
32 # upload benchmarking scripts
33 - cd $VLLM_SOURCE_CODE_LOC/
33 + cd "$VLLM_SOURCE_CODE_LOC/"
34 zip -r nightly-benchmarks.zip .buildkite/ benchmarks/
35 /workspace/buildkite-agent artifact upload "nightly-benchmarks.zip"
36
37 - cd $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/
37 + cd "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/"
38 # upload benchmarking pipeline
39 /workspace/buildkite-agent artifact upload "nightly-pipeline.yaml"
40
41 - cd $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/
41 + cd "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/"
42 /workspace/buildkite-agent annotate --style "success" --context "nightly-benchmarks-results" --append < nightly-annotation.md
43
44
@@ -75,4 +75,4 @@ main() {
75 # /workspace/buildkite-agent annotate --style "success" --context "nightly-benchmarks-results" --append < nightly_results.md
76 }
77
78 -main "$@"
\ No newline at end of file
78 +main "$@"
.buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh
+14 -16
@@ -12,7 +12,7 @@ check_gpus() {
12 echo "Need at least 1 GPU to run benchmarking."
13 exit 1
14 fi
15 - declare -g gpu_type=$(echo $(nvidia-smi --query-gpu=name --format=csv,noheader) | awk '{print $2}')
15 + declare -g gpu_type="$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}')"
16 echo "GPU type is $gpu_type"
17 }
18
@@ -102,7 +102,7 @@ kill_gpu_processes() {
102 pkill -f text-generation
103 pkill -f lmdeploy
104
105 - while [ $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1) -ge 1000 ]; do
105 + while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1)" -ge 1000 ]; do
106 sleep 1
107 done
108 }
@@ -119,8 +119,8 @@ wait_for_server() {
119 ensure_installed() {
120 # Ensure that the given command is installed by apt-get
121 local cmd=$1
122 - if ! which $cmd >/dev/null; then
123 - apt-get update && apt-get install -y $cmd
122 + if ! which "$cmd" >/dev/null; then
123 + apt-get update && apt-get install -y "$cmd"
124 fi
125 }
126
@@ -173,13 +173,11 @@ run_serving_tests() {
173 echo "Reuse previous server for test case $test_name"
174 else
175 kill_gpu_processes
176 - bash $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/launch-server.sh \
176 + bash "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/launch-server.sh" \
177 "$server_params" "$common_params"
178 fi
179
180 - wait_for_server
181 -
182 - if [ $? -eq 0 ]; then
180 + if wait_for_server; then
181 echo ""
182 echo "$CURRENT_LLM_SERVING_ENGINE server is up and running."
183 else
@@ -190,13 +188,13 @@ run_serving_tests() {
188
189 # prepare tokenizer
190 # this is required for lmdeploy.
193 - cd $VLLM_SOURCE_CODE_LOC/benchmarks
191 + cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
192 rm -rf /tokenizer_cache
193 mkdir /tokenizer_cache
194 python3 ../.buildkite/nightly-benchmarks/scripts/download-tokenizer.py \
195 --model "$model" \
196 --cachedir /tokenizer_cache
199 - cd $VLLM_SOURCE_CODE_LOC/benchmarks
197 + cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
198
199
200 # change model name for lmdeploy (it will not follow standard hf name)
@@ -307,11 +305,11 @@ run_serving_tests() {
305 prepare_dataset() {
306
307 # download sharegpt dataset
310 - cd $VLLM_SOURCE_CODE_LOC/benchmarks
308 + cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
309 wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
310
311 # duplicate sonnet by 4x, to allow benchmarking with input length 2048
314 - cd $VLLM_SOURCE_CODE_LOC/benchmarks
312 + cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
313 echo "" > sonnet_4x.txt
314 for _ in {1..4}
315 do
@@ -339,17 +337,17 @@ main() {
337
338 prepare_dataset
339
342 - cd $VLLM_SOURCE_CODE_LOC/benchmarks
340 + cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
341 declare -g RESULTS_FOLDER=results/
342 mkdir -p $RESULTS_FOLDER
345 - BENCHMARK_ROOT=$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/
343 + BENCHMARK_ROOT="$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/"
344
345 # run the test
348 - run_serving_tests $BENCHMARK_ROOT/tests/nightly-tests.json
346 + run_serving_tests "$BENCHMARK_ROOT/tests/nightly-tests.json"
347
348 # upload benchmark results to buildkite
349 python3 -m pip install tabulate pandas
352 - python3 $BENCHMARK_ROOT/scripts/summary-nightly-results.py
350 + python3 "$BENCHMARK_ROOT/scripts/summary-nightly-results.py"
351 upload_to_buildkite
352
353 }
.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
+9 -10
@@ -17,7 +17,7 @@ check_gpus() {
17 echo "Need at least 1 GPU to run benchmarking."
18 exit 1
19 fi
20 - declare -g gpu_type=$(echo $(nvidia-smi --query-gpu=name --format=csv,noheader) | awk '{print $2}')
20 + declare -g gpu_type=$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}')
21 echo "GPU type is $gpu_type"
22 }
23
@@ -93,7 +93,7 @@ kill_gpu_processes() {
93
94
95 # wait until GPU memory usage smaller than 1GB
96 - while [ $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1) -ge 1000 ]; do
96 + while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1)" -ge 1000 ]; do
97 sleep 1
98 done
99
@@ -117,7 +117,7 @@ upload_to_buildkite() {
117 fi
118
119 # Use the determined command to annotate and upload artifacts
120 - $BUILDKITE_AGENT_COMMAND annotate --style "info" --context "$BUILDKITE_LABEL-benchmark-results" <$RESULTS_FOLDER/benchmark_results.md
120 + $BUILDKITE_AGENT_COMMAND annotate --style "info" --context "$BUILDKITE_LABEL-benchmark-results" < "$RESULTS_FOLDER/benchmark_results.md"
121 $BUILDKITE_AGENT_COMMAND artifact upload "$RESULTS_FOLDER/*"
122 }
123
@@ -150,7 +150,7 @@ run_latency_tests() {
150 # check if there is enough GPU to run the test
151 tp=$(echo "$latency_params" | jq -r '.tensor_parallel_size')
152 if [[ $gpu_count -lt $tp ]]; then
153 - echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $testname."
153 + echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $test_name."
154 continue
155 fi
156
@@ -206,9 +206,9 @@ run_throughput_tests() {
206 throughput_args=$(json2args "$throughput_params")
207
208 # check if there is enough GPU to run the test
209 - tp=$(echo $throughput_params | jq -r '.tensor_parallel_size')
209 + tp=$(echo "$throughput_params" | jq -r '.tensor_parallel_size')
210 if [[ $gpu_count -lt $tp ]]; then
211 - echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $testname."
211 + echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $test_name."
212 continue
213 fi
214
@@ -270,7 +270,7 @@ run_serving_tests() {
270 # check if there is enough GPU to run the test
271 tp=$(echo "$server_params" | jq -r '.tensor_parallel_size')
272 if [[ $gpu_count -lt $tp ]]; then
273 - echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $testname."
273 + echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $test_name."
274 continue
275 fi
276
@@ -278,7 +278,7 @@ run_serving_tests() {
278 server_model=$(echo "$server_params" | jq -r '.model')
279 client_model=$(echo "$client_params" | jq -r '.model')
280 if [[ $server_model != "$client_model" ]]; then
281 - echo "Server model and client model must be the same. Skip testcase $testname."
281 + echo "Server model and client model must be the same. Skip testcase $test_name."
282 continue
283 fi
284
@@ -293,8 +293,7 @@ run_serving_tests() {
293 server_pid=$!
294
295 # wait until the server is alive
296 - wait_for_server
297 - if [ $? -eq 0 ]; then
296 + if wait_for_server; then
297 echo ""
298 echo "vllm server is up and running."
299 else
.buildkite/nightly-benchmarks/scripts/wait-for-image.sh
+2 -2
@@ -6,7 +6,7 @@ TIMEOUT_SECONDS=10
6
7 retries=0
8 while [ $retries -lt 1000 ]; do
9 - if [ $(curl -s --max-time $TIMEOUT_SECONDS -L -H "Authorization: Bearer $TOKEN" -o /dev/null -w "%{http_code}" $URL) -eq 200 ]; then
9 + if [ "$(curl -s --max-time "$TIMEOUT_SECONDS" -L -H "Authorization: Bearer $TOKEN" -o /dev/null -w "%{http_code}" "$URL")" -eq 200 ]; then
10 exit 0
11 fi
12
@@ -16,4 +16,4 @@ while [ $retries -lt 1000 ]; do
16 sleep 5
17 done
18
19 -exit 1
\ No newline at end of file
19 +exit 1
.buildkite/run-amd-test.sh
+18 -16
@@ -1,3 +1,5 @@
1 +#!/bin/bash
2 +
3 # This script runs test inside the corresponding ROCm docker container.
4 set -o pipefail
5
@@ -57,17 +59,17 @@ done
59 echo "--- Pulling container"
60 image_name="rocm/vllm-ci:${BUILDKITE_COMMIT}"
61 container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
60 -docker pull ${image_name}
62 +docker pull "${image_name}"
63
64 remove_docker_container() {
63 - docker rm -f ${container_name} || docker image rm -f ${image_name} || true
65 + docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true
66 }
67 trap remove_docker_container EXIT
68
69 echo "--- Running container"
70
71 HF_CACHE="$(realpath ~)/huggingface"
70 -mkdir -p ${HF_CACHE}
72 +mkdir -p "${HF_CACHE}"
73 HF_MOUNT="/root/.cache/huggingface"
74
75 commands=$@
@@ -118,25 +120,25 @@ if [[ $commands == *"--shard-id="* ]]; then
120 --network host \
121 --shm-size=16gb \
122 --rm \
121 - -e HIP_VISIBLE_DEVICES=${GPU} \
123 + -e HIP_VISIBLE_DEVICES="${GPU}" \
124 -e HF_TOKEN \
123 - -v ${HF_CACHE}:${HF_MOUNT} \
124 - -e HF_HOME=${HF_MOUNT} \
125 - --name ${container_name}_${GPU} \
126 - ${image_name} \
125 + -v "${HF_CACHE}:${HF_MOUNT}" \
126 + -e "HF_HOME=${HF_MOUNT}" \
127 + --name "${container_name}_${GPU}" \
128 + "${image_name}" \
129 /bin/bash -c "${commands_gpu}" \
130 |& while read -r line; do echo ">>Shard $GPU: $line"; done &
131 PIDS+=($!)
132 done
133 #wait for all processes to finish and collect exit codes
132 - for pid in ${PIDS[@]}; do
133 - wait ${pid}
134 + for pid in "${PIDS[@]}"; do
135 + wait "${pid}"
136 STATUS+=($?)
137 done
136 - for st in ${STATUS[@]}; do
138 + for st in "${STATUS[@]}"; do
139 if [[ ${st} -ne 0 ]]; then
140 echo "One of the processes failed with $st"
139 - exit ${st}
141 + exit "${st}"
142 fi
143 done
144 else
@@ -147,9 +149,9 @@ else
149 --rm \
150 -e HIP_VISIBLE_DEVICES=0 \
151 -e HF_TOKEN \
150 - -v ${HF_CACHE}:${HF_MOUNT} \
151 - -e HF_HOME=${HF_MOUNT} \
152 - --name ${container_name} \
153 - ${image_name} \
152 + -v "${HF_CACHE}:${HF_MOUNT}" \
153 + -e "HF_HOME=${HF_MOUNT}" \
154 + --name "${container_name}" \
155 + "${image_name}" \
156 /bin/bash -c "${commands}"
157 fi
.buildkite/run-benchmarks.sh
+2
@@ -1,3 +1,5 @@
1 +#!/bin/bash
2 +
3 # This script is run by buildkite to run the benchmarks and upload the results to buildkite
4
5 set -ex
.buildkite/run-cpu-test-ppc64le.sh
+3 -1
@@ -1,3 +1,5 @@
1 +#!/bin/bash
2 +
3 # This script build the CPU docker image and run the offline inference inside the container.
4 # It serves a sanity check for compilation and basic model usage.
5 set -ex
@@ -13,7 +15,7 @@ remove_docker_container
15 # Run the image, setting --shm-size=4g for tensor parallel.
16 source /etc/environment
17 #docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test cpu-test
16 -docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN=$HF_TOKEN --name cpu-test cpu-test
18 +docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN="$HF_TOKEN" --name cpu-test cpu-test
19
20 # Run basic model test
21 docker exec cpu-test bash -c "
.buildkite/run-cpu-test.sh
+2
@@ -1,3 +1,5 @@
1 +#!/bin/bash
2 +
3 # This script build the CPU docker image and run the offline inference inside the container.
4 # It serves a sanity check for compilation and basic model usage.
5 set -ex
.buildkite/run-multi-node-test.sh
+15 -12
@@ -14,7 +14,7 @@ DOCKER_IMAGE=$4
14
15 shift 4
16 COMMANDS=("$@")
17 -if [ ${#COMMANDS[@]} -ne $NUM_NODES ]; then
17 +if [ ${#COMMANDS[@]} -ne "$NUM_NODES" ]; then
18 echo "The number of commands must be equal to the number of nodes."
19 echo "Number of nodes: $NUM_NODES"
20 echo "Number of commands: ${#COMMANDS[@]}"
@@ -23,7 +23,7 @@ fi
23
24 echo "List of commands"
25 for command in "${COMMANDS[@]}"; do
26 - echo $command
26 + echo "$command"
27 done
28
29 start_network() {
@@ -36,7 +36,7 @@ start_nodes() {
36 for node_gpu in $(seq 0 $(($NUM_GPUS - 1))); do
37 DEVICE_NUM=$(($node * $NUM_GPUS + $node_gpu))
38 GPU_DEVICES+=$(($DEVICE_NUM))
39 - if [ $node_gpu -lt $(($NUM_GPUS - 1)) ]; then
39 + if [ "$node_gpu" -lt $(($NUM_GPUS - 1)) ]; then
40 GPU_DEVICES+=','
41 fi
42 done
@@ -49,17 +49,20 @@ start_nodes() {
49 # 3. map the huggingface cache directory to the container
50 # 3. assign ip addresses to the containers (head node: 192.168.10.10, worker nodes:
51 # starting from 192.168.10.11)
52 - docker run -d --gpus "$GPU_DEVICES" --shm-size=10.24gb -e HF_TOKEN -v ~/.cache/huggingface:/root/.cache/huggingface --name node$node --network docker-net --ip 192.168.10.$((10 + $node)) --rm $DOCKER_IMAGE /bin/bash -c "tail -f /dev/null"
52 + docker run -d --gpus "$GPU_DEVICES" --shm-size=10.24gb -e HF_TOKEN \
53 + -v ~/.cache/huggingface:/root/.cache/huggingface --name "node$node" \
54 + --network docker-net --ip 192.168.10.$((10 + $node)) --rm "$DOCKER_IMAGE" \
55 + /bin/bash -c "tail -f /dev/null"
56
57 # organize containers into a ray cluster
55 - if [ $node -eq 0 ]; then
58 + if [ "$node" -eq 0 ]; then
59 # start the ray head node
57 - docker exec -d node$node /bin/bash -c "ray start --head --port=6379 --block"
60 + docker exec -d "node$node" /bin/bash -c "ray start --head --port=6379 --block"
61 # wait for the head node to be ready
62 sleep 10
63 else
64 # start the ray worker nodes, and connect them to the head node
62 - docker exec -d node$node /bin/bash -c "ray start --address=192.168.10.10:6379 --block"
65 + docker exec -d "node$node" /bin/bash -c "ray start --address=192.168.10.10:6379 --block"
66 fi
67 done
68
@@ -79,22 +82,22 @@ run_nodes() {
82 for node_gpu in $(seq 0 $(($NUM_GPUS - 1))); do
83 DEVICE_NUM=$(($node * $NUM_GPUS + $node_gpu))
84 GPU_DEVICES+=$(($DEVICE_NUM))
82 - if [ $node_gpu -lt $(($NUM_GPUS - 1)) ]; then
85 + if [ "$node_gpu" -lt $(($NUM_GPUS - 1)) ]; then
86 GPU_DEVICES+=','
87 fi
88 done
89 GPU_DEVICES+='"'
90 echo "Running node$node with GPU devices: $GPU_DEVICES"
88 - if [ $node -ne 0 ]; then
89 - docker exec -d node$node /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
91 + if [ "$node" -ne 0 ]; then
92 + docker exec -d "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
93 else
91 - docker exec node$node /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
94 + docker exec "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
95 fi
96 done
97 }
98 cleanup() {
99 for node in $(seq 0 $(($NUM_NODES-1))); do
97 - docker stop node$node
100 + docker stop "node$node"
101 done
102 docker network rm docker-net
103 }
.buildkite/run-neuron-test.sh
+5 -3
@@ -1,3 +1,5 @@
1 +#!/bin/bash
2 +
3 # This script build the Neuron docker image and run the API server inside the container.
4 # It serves a sanity check for compilation and basic model usage.
5 set -e
@@ -12,10 +14,10 @@ if [ -f /tmp/neuron-docker-build-timestamp ]; then
14 current_time=$(date +%s)
15 if [ $((current_time - last_build)) -gt 86400 ]; then
16 docker system prune -f
15 - echo $current_time > /tmp/neuron-docker-build-timestamp
17 + echo "$current_time" > /tmp/neuron-docker-build-timestamp
18 fi
19 else
18 - echo $(date +%s) > /tmp/neuron-docker-build-timestamp
20 + date "+%s" > /tmp/neuron-docker-build-timestamp
21 fi
22
23 docker build -t neuron -f Dockerfile.neuron .
@@ -34,7 +36,7 @@ wait_for_server_to_start() {
36 timeout=300
37 counter=0
38
37 - while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8000/health)" != "200" ]; do
39 + while [ "$(curl -s -o /dev/null -w '%{http_code}' localhost:8000/health)" != "200" ]; do
40 sleep 1
41 counter=$((counter + 1))
42 if [ $counter -ge $timeout ]; then
.buildkite/run-openvino-test.sh
+2
@@ -1,3 +1,5 @@
1 +#!/bin/bash
2 +
3 # This script build the OpenVINO docker image and run the offline inference inside the container.
4 # It serves a sanity check for compilation and basic model usage.
5 set -ex
.buildkite/run-tpu-test.sh
+3 -1
@@ -1,3 +1,5 @@
1 +#!/bin/bash
2 +
3 set -e
4
5 # Build the docker image.
@@ -12,4 +14,4 @@ remove_docker_container
14 # For HF_TOKEN.
15 source /etc/environment
16 # Run a simple end-to-end example.
15 -docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && python3 -m pip install lm_eval[api]==0.4.4 && pytest -v -s /workspace/vllm/tests/entrypoints/openai/test_accuracy.py && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
17 +docker run --privileged --net host --shm-size=16G -it -e "HF_TOKEN=$HF_TOKEN" --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && python3 -m pip install lm_eval[api]==0.4.4 && pytest -v -s /workspace/vllm/tests/entrypoints/openai/test_accuracy.py && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
.buildkite/run-xpu-test.sh
+2
@@ -1,3 +1,5 @@
1 +#!/bin/bash
2 +
3 # This script build the CPU docker image and run the offline inference inside the container.
4 # It serves a sanity check for compilation and basic model usage.
5 set -ex
.github/workflows/scripts/cuda-install.sh
+4 -4
@@ -1,16 +1,16 @@
1 #!/bin/bash
2
3 # Replace '.' with '-' ex: 11.8 -> 11-8
4 -cuda_version=$(echo $1 | tr "." "-")
4 +cuda_version=$(echo "$1" | tr "." "-")
5 # Removes '-' and '.' ex: ubuntu-20.04 -> ubuntu2004
6 -OS=$(echo $2 | tr -d ".\-")
6 +OS=$(echo "$2" | tr -d ".\-")
7
8 # Installs CUDA
9 -wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-keyring_1.1-1_all.deb
9 +wget -nv "https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-keyring_1.1-1_all.deb"
10 sudo dpkg -i cuda-keyring_1.1-1_all.deb
11 rm cuda-keyring_1.1-1_all.deb
12 sudo apt -qq update
13 -sudo apt -y install cuda-${cuda_version} cuda-nvcc-${cuda_version} cuda-libraries-dev-${cuda_version}
13 +sudo apt -y install "cuda-${cuda_version}" "cuda-nvcc-${cuda_version}" "cuda-libraries-dev-${cuda_version}"
14 sudo apt clean
15
16 # Test nvcc
.github/workflows/scripts/pytorch-install.sh
+1 -1
@@ -6,7 +6,7 @@ cuda_version=$3
6
7 # Install torch
8 $python_executable -m pip install numpy pyyaml scipy ipython mkl mkl-include ninja cython typing pandas typing-extensions dataclasses setuptools && conda clean -ya
9 -$python_executable -m pip install torch==${pytorch_version}+cu${cuda_version//./} --extra-index-url https://download.pytorch.org/whl/cu${cuda_version//./}
9 +$python_executable -m pip install torch=="${pytorch_version}+cu${cuda_version//./}" --extra-index-url "https://download.pytorch.org/whl/cu${cuda_version//./}"
10
11 # Print version information
12 $python_executable --version
.github/workflows/shellcheck.yml new
+37
@@ -0,0 +1,37 @@
1 +name: Lint shell scripts
2 +on:
3 + push:
4 + branches:
5 + - "main"
6 + paths:
7 + - '**/*.sh'
8 + - '.github/workflows/shellcheck.yml'
9 + pull_request:
10 + branches:
11 + - "main"
12 + paths:
13 + - '**/*.sh'
14 + - '.github/workflows/shellcheck.yml'
15 +
16 +env:
17 + LC_ALL: en_US.UTF-8
18 +
19 +defaults:
20 + run:
21 + shell: bash
22 +
23 +permissions:
24 + contents: read
25 +
26 +jobs:
27 + shellcheck:
28 + runs-on: ubuntu-latest
29 + steps:
30 + - name: "Checkout"
31 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
32 + with:
33 + fetch-depth: 0
34 +
35 + - name: "Check shell scripts"
36 + run: |
37 + tools/shellcheck.sh
.gitignore
+1
@@ -202,3 +202,4 @@ benchmarks/*.json
202
203 # Linting
204 actionlint
205 +shellcheck*/
.shellcheckrc new
+9
@@ -0,0 +1,9 @@
1 +# rules currently disabled:
2 +#
3 +# SC1091 (info): Not following: <sourced file> was not specified as input (see shellcheck -x)
4 +# SC2004 (style): $/${} is unnecessary on arithmetic variables.
5 +# SC2129 (style): Consider using { cmd1; cmd2; } >> file instead of individual redirects.
6 +# SC2155 (warning): Declare and assign separately to avoid masking return values.
7 +# SC2164 (warning): Use 'cd ... || exit' or 'cd ... || return' in case cd fails.
8 +#
9 +disable=SC1091,SC2004,SC2129,SC2155,SC2164
benchmarks/launch_tgi_server.sh
+4 -4
@@ -4,13 +4,13 @@ PORT=8000
4 MODEL=$1
5 TOKENS=$2
6
7 -docker run -e HF_TOKEN=$HF_TOKEN --gpus all --shm-size 1g -p $PORT:80 \
8 - -v $PWD/data:/data \
7 +docker run -e "HF_TOKEN=$HF_TOKEN" --gpus all --shm-size 1g -p $PORT:80 \
8 + -v "$PWD/data:/data" \
9 ghcr.io/huggingface/text-generation-inference:2.2.0 \
10 - --model-id $MODEL \
10 + --model-id "$MODEL" \
11 --sharded false \
12 --max-input-length 1024 \
13 --max-total-tokens 2048 \
14 --max-best-of 5 \
15 --max-concurrent-requests 5000 \
16 - --max-batch-total-tokens $TOKENS
16 + --max-batch-total-tokens "$TOKENS"
examples/run_cluster.sh
+2 -2
@@ -14,7 +14,7 @@ PATH_TO_HF_HOME="$4"
14 shift 4
15
16 # Additional arguments are passed directly to the Docker command
17 -ADDITIONAL_ARGS="$@"
17 +ADDITIONAL_ARGS=("$@")
18
19 # Validate node type
20 if [ "${NODE_TYPE}" != "--head" ] && [ "${NODE_TYPE}" != "--worker" ]; then
@@ -45,5 +45,5 @@ docker run \
45 --shm-size 10.24g \
46 --gpus all \
47 -v "${PATH_TO_HF_HOME}:/root/.cache/huggingface" \
48 - ${ADDITIONAL_ARGS} \
48 + "${ADDITIONAL_ARGS[@]}" \
49 "${DOCKER_IMAGE}" -c "${RAY_START_CMD}"
format.sh
+7 -3
@@ -44,14 +44,14 @@ CLANGFORMAT_VERSION=$(clang-format --version | awk '{print $3}')
44
45 # # params: tool name, tool version, required version
46 tool_version_check() {
47 - if [[ $2 != $3 ]]; then
47 + if [[ "$2" != "$3" ]]; then
48 echo "❓❓Wrong $1 version installed: $3 is required, not $2."
49 exit 1
50 fi
51 }
52
53 -tool_version_check "yapf" $YAPF_VERSION "$(grep yapf requirements-lint.txt | cut -d'=' -f3)"
54 -tool_version_check "ruff" $RUFF_VERSION "$(grep "ruff==" requirements-lint.txt | cut -d'=' -f3)"
53 +tool_version_check "yapf" "$YAPF_VERSION" "$(grep yapf requirements-lint.txt | cut -d'=' -f3)"
54 +tool_version_check "ruff" "$RUFF_VERSION" "$(grep "ruff==" requirements-lint.txt | cut -d'=' -f3)"
55 tool_version_check "mypy" "$MYPY_VERSION" "$(grep mypy requirements-lint.txt | cut -d'=' -f3)"
56 tool_version_check "isort" "$ISORT_VERSION" "$(grep isort requirements-lint.txt | cut -d'=' -f3)"
57 tool_version_check "codespell" "$CODESPELL_VERSION" "$(grep codespell requirements-lint.txt | cut -d'=' -f3)"
@@ -294,6 +294,10 @@ echo 'vLLM actionlint:'
294 tools/actionlint.sh -color
295 echo 'vLLM actionlint: Done'
296
297 +echo 'vLLM shellcheck:'
298 +tools/shellcheck.sh
299 +echo 'vLLM shellcheck: Done'
300 +
301 if ! git diff --quiet &>/dev/null; then
302 echo
303 echo "🔍🔍There are files changed by the format checker or by you that are not added and committed:"
tests/weight_loading/run_model_weight_loading_test.sh
+1 -1
@@ -14,7 +14,7 @@ while getopts "c:" OPT; do
14 done
15
16
17 -IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < $CONFIG
17 +IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "$CONFIG"
18
19 for MODEL_CONFIG in "${MODEL_CONFIGS[@]}"
20 do
tools/mypy.sh
+2 -2
@@ -3,13 +3,13 @@
3 CI=${1:-0}
4 PYTHON_VERSION=${2:-3.9}
5
6 -if [ $CI -eq 1 ]; then
6 +if [ "$CI" -eq 1 ]; then
7 set -e
8 fi
9
10 run_mypy() {
11 echo "Running mypy on $1"
12 - if [ $CI -eq 1 ] && [ -z "$1" ]; then
12 + if [ "$CI" -eq 1 ] && [ -z "$1" ]; then
13 mypy --python-version "${PYTHON_VERSION}" "$@"
14 return
15 fi
tools/shellcheck.sh new
+21
@@ -0,0 +1,21 @@
1 +#!/bin/bash
2 +
3 +scversion="stable"
4 +
5 +if [ -d "shellcheck-${scversion}" ]; then
6 + export PATH="$PATH:$(pwd)/shellcheck-${scversion}"
7 +fi
8 +
9 +if ! [ -x "$(command -v shellcheck)" ]; then
10 + if [ "$(uname -s)" != "Linux" ] || [ "$(uname -m)" != "x86_64" ]; then
11 + echo "Please install shellcheck: https://github.com/koalaman/shellcheck?tab=readme-ov-file#installing"
12 + exit 1
13 + fi
14 +
15 + # automatic local install if linux x86_64
16 + wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${scversion?}/shellcheck-${scversion?}.linux.x86_64.tar.xz" | tar -xJv
17 + export PATH="$PATH:$(pwd)/shellcheck-${scversion}"
18 +fi
19 +
20 +# TODO - fix warnings in .buildkite/run-amd-test.sh
21 +find . -name "*.sh" -not -path "./.deps/*" -not -path "./.buildkite/run-amd-test.sh" -exec shellcheck {} +