A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[CI/Build] Add shell script linting using shellcheck (#7925)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Nov 7, 2024 at 13:17 UTC
3be5b26a7651b57aeb2cbdfc6aee81152ba68da5
28 files changed
+204
-129
.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh
+3
-3
@@ -41,6 +41,6 @@ while getopts "m:b:l:f:" OPT; do
41
done
42
43
lm_eval --model hf \
44
- --model_args pretrained=$MODEL,parallelize=True \
45
- --tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
46
- --batch_size $BATCH_SIZE
44
+ --model_args "pretrained=$MODEL,parallelize=True" \
45
+ --tasks gsm8k --num_fewshot "$FEWSHOT" --limit "$LIMIT" \
46
+ --batch_size "$BATCH_SIZE"
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+3
-3
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
46
done
47
48
lm_eval --model vllm \
49
- --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,distributed_executor_backend="ray",trust_remote_code=true,max_model_len=4096 \
50
- --tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
51
- --batch_size $BATCH_SIZE
49
+ --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,distributed_executor_backend=ray,trust_remote_code=true,max_model_len=4096" \
50
+ --tasks gsm8k --num_fewshot "$FEWSHOT" --limit "$LIMIT" \
51
+ --batch_size "$BATCH_SIZE"
.buildkite/lm-eval-harness/run-tests.sh
+1
-1
@@ -30,7 +30,7 @@ while getopts "c:t:" OPT; do
30
done
31
32
# Parse list of configs.
33
-IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < $CONFIG
33
+IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "$CONFIG"
34
35
for MODEL_CONFIG in "${MODEL_CONFIGS[@]}"
36
do
.buildkite/nightly-benchmarks/scripts/launch-server.sh
+25
-38
@@ -50,31 +50,30 @@ launch_trt_server() {
50
git clone https://github.com/triton-inference-server/tensorrtllm_backend.git
51
git lfs install
52
cd tensorrtllm_backend
53
- git checkout $trt_llm_version
54
- tensorrtllm_backend_dir=$(pwd)
53
+ git checkout "$trt_llm_version"
54
git submodule update --init --recursive
55
56
# build trtllm engine
57
cd /tensorrtllm_backend
59
- cd ./tensorrt_llm/examples/${model_type}
58
+ cd "./tensorrt_llm/examples/${model_type}"
59
python3 convert_checkpoint.py \
61
- --model_dir ${model_path} \
62
- --dtype ${model_dtype} \
63
- --tp_size ${model_tp_size} \
64
- --output_dir ${trt_model_path}
60
+ --model_dir "${model_path}" \
61
+ --dtype "${model_dtype}" \
62
+ --tp_size "${model_tp_size}" \
63
+ --output_dir "${trt_model_path}"
64
trtllm-build \
66
- --checkpoint_dir ${trt_model_path} \
65
+ --checkpoint_dir "${trt_model_path}" \
66
--use_fused_mlp \
67
--reduce_fusion disable \
68
--workers 8 \
70
- --gpt_attention_plugin ${model_dtype} \
71
- --gemm_plugin ${model_dtype} \
72
- --tp_size ${model_tp_size} \
73
- --max_batch_size ${max_batch_size} \
74
- --max_input_len ${max_input_len} \
75
- --max_seq_len ${max_seq_len} \
76
- --max_num_tokens ${max_num_tokens} \
77
- --output_dir ${trt_engine_path}
69
+ --gpt_attention_plugin "${model_dtype}" \
70
+ --gemm_plugin "${model_dtype}" \
71
+ --tp_size "${model_tp_size}" \
72
+ --max_batch_size "${max_batch_size}" \
73
+ --max_input_len "${max_input_len}" \
74
+ --max_seq_len "${max_seq_len}" \
75
+ --max_num_tokens "${max_num_tokens}" \
76
+ --output_dir "${trt_engine_path}"
77
78
# handle triton protobuf files and launch triton server
79
cd /tensorrtllm_backend
@@ -82,15 +81,15 @@ launch_trt_server() {
81
cp -r all_models/inflight_batcher_llm/* triton_model_repo/
82
cd triton_model_repo
83
rm -rf ./tensorrt_llm/1/*
85
- cp -r ${trt_engine_path}/* ./tensorrt_llm/1
84
+ cp -r "${trt_engine_path}"/* ./tensorrt_llm/1
85
python3 ../tools/fill_template.py -i tensorrt_llm/config.pbtxt triton_backend:tensorrtllm,engine_dir:/tensorrtllm_backend/triton_model_repo/tensorrt_llm/1,decoupled_mode:true,batching_strategy:inflight_fused_batching,batch_scheduler_policy:guaranteed_no_evict,exclude_input_in_output:true,triton_max_batch_size:2048,max_queue_delay_microseconds:0,max_beam_width:1,max_queue_size:2048,enable_kv_cache_reuse:false
87
- python3 ../tools/fill_template.py -i preprocessing/config.pbtxt triton_max_batch_size:2048,tokenizer_dir:$model_path,preprocessing_instance_count:5
88
- python3 ../tools/fill_template.py -i postprocessing/config.pbtxt triton_max_batch_size:2048,tokenizer_dir:$model_path,postprocessing_instance_count:5,skip_special_tokens:false
89
- python3 ../tools/fill_template.py -i ensemble/config.pbtxt triton_max_batch_size:$max_batch_size
90
- python3 ../tools/fill_template.py -i tensorrt_llm_bls/config.pbtxt triton_max_batch_size:$max_batch_size,decoupled_mode:true,accumulate_tokens:"False",bls_instance_count:1
86
+ python3 ../tools/fill_template.py -i preprocessing/config.pbtxt "triton_max_batch_size:2048,tokenizer_dir:$model_path,preprocessing_instance_count:5"
87
+ python3 ../tools/fill_template.py -i postprocessing/config.pbtxt "triton_max_batch_size:2048,tokenizer_dir:$model_path,postprocessing_instance_count:5,skip_special_tokens:false"
88
+ python3 ../tools/fill_template.py -i ensemble/config.pbtxt triton_max_batch_size:"$max_batch_size"
89
+ python3 ../tools/fill_template.py -i tensorrt_llm_bls/config.pbtxt "triton_max_batch_size:$max_batch_size,decoupled_mode:true,accumulate_tokens:False,bls_instance_count:1"
90
cd /tensorrtllm_backend
91
python3 scripts/launch_triton_server.py \
93
- --world_size=${model_tp_size} \
92
+ --world_size="${model_tp_size}" \
93
--model_repo=/tensorrtllm_backend/triton_model_repo &
94
95
}
@@ -98,10 +97,7 @@ launch_trt_server() {
97
launch_tgi_server() {
98
model=$(echo "$common_params" | jq -r '.model')
99
tp=$(echo "$common_params" | jq -r '.tp')
101
- dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
102
- dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
100
port=$(echo "$common_params" | jq -r '.port')
104
- num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
101
server_args=$(json2args "$server_params")
102
103
if echo "$common_params" | jq -e 'has("fp8")' >/dev/null; then
@@ -129,10 +125,7 @@ launch_tgi_server() {
125
launch_lmdeploy_server() {
126
model=$(echo "$common_params" | jq -r '.model')
127
tp=$(echo "$common_params" | jq -r '.tp')
132
- dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
133
- dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
128
port=$(echo "$common_params" | jq -r '.port')
135
- num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
129
server_args=$(json2args "$server_params")
130
131
server_command="lmdeploy serve api_server $model \
@@ -149,10 +142,7 @@ launch_sglang_server() {
142
143
model=$(echo "$common_params" | jq -r '.model')
144
tp=$(echo "$common_params" | jq -r '.tp')
152
- dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
153
- dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
145
port=$(echo "$common_params" | jq -r '.port')
155
- num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
146
server_args=$(json2args "$server_params")
147
148
if echo "$common_params" | jq -e 'has("fp8")' >/dev/null; then
@@ -185,10 +175,7 @@ launch_vllm_server() {
175
176
model=$(echo "$common_params" | jq -r '.model')
177
tp=$(echo "$common_params" | jq -r '.tp')
188
- dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
189
- dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
178
port=$(echo "$common_params" | jq -r '.port')
191
- num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
179
server_args=$(json2args "$server_params")
180
181
if echo "$common_params" | jq -e 'has("fp8")' >/dev/null; then
@@ -217,19 +204,19 @@ launch_vllm_server() {
204
205
main() {
206
220
- if [[ $CURRENT_LLM_SERVING_ENGINE == "trt" ]]; then
207
+ if [[ "$CURRENT_LLM_SERVING_ENGINE" == "trt" ]]; then
208
launch_trt_server
209
fi
210
224
- if [[ $CURRENT_LLM_SERVING_ENGINE == "tgi" ]]; then
211
+ if [[ "$CURRENT_LLM_SERVING_ENGINE" == "tgi" ]]; then
212
launch_tgi_server
213
fi
214
228
- if [[ $CURRENT_LLM_SERVING_ENGINE == "lmdeploy" ]]; then
215
+ if [[ "$CURRENT_LLM_SERVING_ENGINE" == "lmdeploy" ]]; then
216
launch_lmdeploy_server
217
fi
218
232
- if [[ $CURRENT_LLM_SERVING_ENGINE == "sglang" ]]; then
219
+ if [[ "$CURRENT_LLM_SERVING_ENGINE" == "sglang" ]]; then
220
launch_sglang_server
221
fi
222
.buildkite/nightly-benchmarks/scripts/nightly-annotate.sh
+6
-6
@@ -16,10 +16,10 @@ main() {
16
fi
17
18
# initial annotation
19
- description="$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/nightly-descriptions.md"
19
+ #description="$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/nightly-descriptions.md"
20
21
# download results
22
- cd $VLLM_SOURCE_CODE_LOC/benchmarks
22
+ cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
23
mkdir -p results/
24
/workspace/buildkite-agent artifact download 'results/*nightly_results.json' results/
25
ls
@@ -30,15 +30,15 @@ main() {
30
/workspace/buildkite-agent artifact upload "results.zip"
31
32
# upload benchmarking scripts
33
- cd $VLLM_SOURCE_CODE_LOC/
33
+ cd "$VLLM_SOURCE_CODE_LOC/"
34
zip -r nightly-benchmarks.zip .buildkite/ benchmarks/
35
/workspace/buildkite-agent artifact upload "nightly-benchmarks.zip"
36
37
- cd $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/
37
+ cd "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/"
38
# upload benchmarking pipeline
39
/workspace/buildkite-agent artifact upload "nightly-pipeline.yaml"
40
41
- cd $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/
41
+ cd "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/"
42
/workspace/buildkite-agent annotate --style "success" --context "nightly-benchmarks-results" --append < nightly-annotation.md
43
44
@@ -75,4 +75,4 @@ main() {
75
# /workspace/buildkite-agent annotate --style "success" --context "nightly-benchmarks-results" --append < nightly_results.md
76
}
77
78
-main "$@"
\ No newline at end of file
78
+main "$@"
.buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh
+14
-16
@@ -12,7 +12,7 @@ check_gpus() {
12
echo "Need at least 1 GPU to run benchmarking."
13
exit 1
14
fi
15
- declare -g gpu_type=$(echo $(nvidia-smi --query-gpu=name --format=csv,noheader) | awk '{print $2}')
15
+ declare -g gpu_type="$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}')"
16
echo "GPU type is $gpu_type"
17
}
18
@@ -102,7 +102,7 @@ kill_gpu_processes() {
102
pkill -f text-generation
103
pkill -f lmdeploy
104
105
- while [ $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1) -ge 1000 ]; do
105
+ while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1)" -ge 1000 ]; do
106
sleep 1
107
done
108
}
@@ -119,8 +119,8 @@ wait_for_server() {
119
ensure_installed() {
120
# Ensure that the given command is installed by apt-get
121
local cmd=$1
122
- if ! which $cmd >/dev/null; then
123
- apt-get update && apt-get install -y $cmd
122
+ if ! which "$cmd" >/dev/null; then
123
+ apt-get update && apt-get install -y "$cmd"
124
fi
125
}
126
@@ -173,13 +173,11 @@ run_serving_tests() {
173
echo "Reuse previous server for test case $test_name"
174
else
175
kill_gpu_processes
176
- bash $VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/launch-server.sh \
176
+ bash "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/launch-server.sh" \
177
"$server_params" "$common_params"
178
fi
179
180
- wait_for_server
181
-
182
- if [ $? -eq 0 ]; then
180
+ if wait_for_server; then
181
echo ""
182
echo "$CURRENT_LLM_SERVING_ENGINE server is up and running."
183
else
@@ -190,13 +188,13 @@ run_serving_tests() {
188
189
# prepare tokenizer
190
# this is required for lmdeploy.
193
- cd $VLLM_SOURCE_CODE_LOC/benchmarks
191
+ cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
192
rm -rf /tokenizer_cache
193
mkdir /tokenizer_cache
194
python3 ../.buildkite/nightly-benchmarks/scripts/download-tokenizer.py \
195
--model "$model" \
196
--cachedir /tokenizer_cache
199
- cd $VLLM_SOURCE_CODE_LOC/benchmarks
197
+ cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
198
199
200
# change model name for lmdeploy (it will not follow standard hf name)
@@ -307,11 +305,11 @@ run_serving_tests() {
305
prepare_dataset() {
306
307
# download sharegpt dataset
310
- cd $VLLM_SOURCE_CODE_LOC/benchmarks
308
+ cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
309
wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
310
311
# duplicate sonnet by 4x, to allow benchmarking with input length 2048
314
- cd $VLLM_SOURCE_CODE_LOC/benchmarks
312
+ cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
313
echo "" > sonnet_4x.txt
314
for _ in {1..4}
315
do
@@ -339,17 +337,17 @@ main() {
337
338
prepare_dataset
339
342
- cd $VLLM_SOURCE_CODE_LOC/benchmarks
340
+ cd "$VLLM_SOURCE_CODE_LOC/benchmarks"
341
declare -g RESULTS_FOLDER=results/
342
mkdir -p $RESULTS_FOLDER
345
- BENCHMARK_ROOT=$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/
343
+ BENCHMARK_ROOT="$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/"
344
345
# run the test
348
- run_serving_tests $BENCHMARK_ROOT/tests/nightly-tests.json
346
+ run_serving_tests "$BENCHMARK_ROOT/tests/nightly-tests.json"
347
348
# upload benchmark results to buildkite
349
python3 -m pip install tabulate pandas
352
- python3 $BENCHMARK_ROOT/scripts/summary-nightly-results.py
350
+ python3 "$BENCHMARK_ROOT/scripts/summary-nightly-results.py"
351
upload_to_buildkite
352
353
}
.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
+9
-10
@@ -17,7 +17,7 @@ check_gpus() {
17
echo "Need at least 1 GPU to run benchmarking."
18
exit 1
19
fi
20
- declare -g gpu_type=$(echo $(nvidia-smi --query-gpu=name --format=csv,noheader) | awk '{print $2}')
20
+ declare -g gpu_type=$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}')
21
echo "GPU type is $gpu_type"
22
}
23
@@ -93,7 +93,7 @@ kill_gpu_processes() {
93
94
95
# wait until GPU memory usage smaller than 1GB
96
- while [ $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1) -ge 1000 ]; do
96
+ while [ "$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -n 1)" -ge 1000 ]; do
97
sleep 1
98
done
99
@@ -117,7 +117,7 @@ upload_to_buildkite() {
117
fi
118
119
# Use the determined command to annotate and upload artifacts
120
- $BUILDKITE_AGENT_COMMAND annotate --style "info" --context "$BUILDKITE_LABEL-benchmark-results" <$RESULTS_FOLDER/benchmark_results.md
120
+ $BUILDKITE_AGENT_COMMAND annotate --style "info" --context "$BUILDKITE_LABEL-benchmark-results" < "$RESULTS_FOLDER/benchmark_results.md"
121
$BUILDKITE_AGENT_COMMAND artifact upload "$RESULTS_FOLDER/*"
122
}
123
@@ -150,7 +150,7 @@ run_latency_tests() {
150
# check if there is enough GPU to run the test
151
tp=$(echo "$latency_params" | jq -r '.tensor_parallel_size')
152
if [[ $gpu_count -lt $tp ]]; then
153
- echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $testname."
153
+ echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $test_name."
154
continue
155
fi
156
@@ -206,9 +206,9 @@ run_throughput_tests() {
206
throughput_args=$(json2args "$throughput_params")
207
208
# check if there is enough GPU to run the test
209
- tp=$(echo $throughput_params | jq -r '.tensor_parallel_size')
209
+ tp=$(echo "$throughput_params" | jq -r '.tensor_parallel_size')
210
if [[ $gpu_count -lt $tp ]]; then
211
- echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $testname."
211
+ echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $test_name."
212
continue
213
fi
214
@@ -270,7 +270,7 @@ run_serving_tests() {
270
# check if there is enough GPU to run the test
271
tp=$(echo "$server_params" | jq -r '.tensor_parallel_size')
272
if [[ $gpu_count -lt $tp ]]; then
273
- echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $testname."
273
+ echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $test_name."
274
continue
275
fi
276
@@ -278,7 +278,7 @@ run_serving_tests() {
278
server_model=$(echo "$server_params" | jq -r '.model')
279
client_model=$(echo "$client_params" | jq -r '.model')
280
if [[ $server_model != "$client_model" ]]; then
281
- echo "Server model and client model must be the same. Skip testcase $testname."
281
+ echo "Server model and client model must be the same. Skip testcase $test_name."
282
continue
283
fi
284
@@ -293,8 +293,7 @@ run_serving_tests() {
293
server_pid=$!
294
295
# wait until the server is alive
296
- wait_for_server
297
- if [ $? -eq 0 ]; then
296
+ if wait_for_server; then
297
echo ""
298
echo "vllm server is up and running."
299
else
.buildkite/nightly-benchmarks/scripts/wait-for-image.sh
+2
-2
@@ -6,7 +6,7 @@ TIMEOUT_SECONDS=10
6
7
retries=0
8
while [ $retries -lt 1000 ]; do
9
- if [ $(curl -s --max-time $TIMEOUT_SECONDS -L -H "Authorization: Bearer $TOKEN" -o /dev/null -w "%{http_code}" $URL) -eq 200 ]; then
9
+ if [ "$(curl -s --max-time "$TIMEOUT_SECONDS" -L -H "Authorization: Bearer $TOKEN" -o /dev/null -w "%{http_code}" "$URL")" -eq 200 ]; then
10
exit 0
11
fi
12
@@ -16,4 +16,4 @@ while [ $retries -lt 1000 ]; do
16
sleep 5
17
done
18
19
-exit 1
\ No newline at end of file
19
+exit 1
.buildkite/run-amd-test.sh
+18
-16
@@ -1,3 +1,5 @@
1
+#!/bin/bash
2
+
3
# This script runs test inside the corresponding ROCm docker container.
4
set -o pipefail
5
@@ -57,17 +59,17 @@ done
59
echo "--- Pulling container"
60
image_name="rocm/vllm-ci:${BUILDKITE_COMMIT}"
61
container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
60
-docker pull ${image_name}
62
+docker pull "${image_name}"
63
64
remove_docker_container() {
63
- docker rm -f ${container_name} || docker image rm -f ${image_name} || true
65
+ docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true
66
}
67
trap remove_docker_container EXIT
68
69
echo "--- Running container"
70
71
HF_CACHE="$(realpath ~)/huggingface"
70
-mkdir -p ${HF_CACHE}
72
+mkdir -p "${HF_CACHE}"
73
HF_MOUNT="/root/.cache/huggingface"
74
75
commands=$@
@@ -118,25 +120,25 @@ if [[ $commands == *"--shard-id="* ]]; then
120
--network host \
121
--shm-size=16gb \
122
--rm \
121
- -e HIP_VISIBLE_DEVICES=${GPU} \
123
+ -e HIP_VISIBLE_DEVICES="${GPU}" \
124
-e HF_TOKEN \
123
- -v ${HF_CACHE}:${HF_MOUNT} \
124
- -e HF_HOME=${HF_MOUNT} \
125
- --name ${container_name}_${GPU} \
126
- ${image_name} \
125
+ -v "${HF_CACHE}:${HF_MOUNT}" \
126
+ -e "HF_HOME=${HF_MOUNT}" \
127
+ --name "${container_name}_${GPU}" \
128
+ "${image_name}" \
129
/bin/bash -c "${commands_gpu}" \
130
|& while read -r line; do echo ">>Shard $GPU: $line"; done &
131
PIDS+=($!)
132
done
133
#wait for all processes to finish and collect exit codes
132
- for pid in ${PIDS[@]}; do
133
- wait ${pid}
134
+ for pid in "${PIDS[@]}"; do
135
+ wait "${pid}"
136
STATUS+=($?)
137
done
136
- for st in ${STATUS[@]}; do
138
+ for st in "${STATUS[@]}"; do
139
if [[ ${st} -ne 0 ]]; then
140
echo "One of the processes failed with $st"
139
- exit ${st}
141
+ exit "${st}"
142
fi
143
done
144
else
@@ -147,9 +149,9 @@ else
149
--rm \
150
-e HIP_VISIBLE_DEVICES=0 \
151
-e HF_TOKEN \
150
- -v ${HF_CACHE}:${HF_MOUNT} \
151
- -e HF_HOME=${HF_MOUNT} \
152
- --name ${container_name} \
153
- ${image_name} \
152
+ -v "${HF_CACHE}:${HF_MOUNT}" \
153
+ -e "HF_HOME=${HF_MOUNT}" \
154
+ --name "${container_name}" \
155
+ "${image_name}" \
156
/bin/bash -c "${commands}"
157
fi
.buildkite/run-benchmarks.sh
+2
@@ -1,3 +1,5 @@
1
+#!/bin/bash
2
+
3
# This script is run by buildkite to run the benchmarks and upload the results to buildkite
4
5
set -ex
.buildkite/run-cpu-test-ppc64le.sh
+3
-1
@@ -1,3 +1,5 @@
1
+#!/bin/bash
2
+
3
# This script build the CPU docker image and run the offline inference inside the container.
4
# It serves a sanity check for compilation and basic model usage.
5
set -ex
@@ -13,7 +15,7 @@ remove_docker_container
15
# Run the image, setting --shm-size=4g for tensor parallel.
16
source /etc/environment
17
#docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test cpu-test
16
-docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN=$HF_TOKEN --name cpu-test cpu-test
18
+docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN="$HF_TOKEN" --name cpu-test cpu-test
19
20
# Run basic model test
21
docker exec cpu-test bash -c "
.buildkite/run-cpu-test.sh
+2
@@ -1,3 +1,5 @@
1
+#!/bin/bash
2
+
3
# This script build the CPU docker image and run the offline inference inside the container.
4
# It serves a sanity check for compilation and basic model usage.
5
set -ex
.buildkite/run-multi-node-test.sh
+15
-12
@@ -14,7 +14,7 @@ DOCKER_IMAGE=$4
14
15
shift 4
16
COMMANDS=("$@")
17
-if [ ${#COMMANDS[@]} -ne $NUM_NODES ]; then
17
+if [ ${#COMMANDS[@]} -ne "$NUM_NODES" ]; then
18
echo "The number of commands must be equal to the number of nodes."
19
echo "Number of nodes: $NUM_NODES"
20
echo "Number of commands: ${#COMMANDS[@]}"
@@ -23,7 +23,7 @@ fi
23
24
echo "List of commands"
25
for command in "${COMMANDS[@]}"; do
26
- echo $command
26
+ echo "$command"
27
done
28
29
start_network() {
@@ -36,7 +36,7 @@ start_nodes() {
36
for node_gpu in $(seq 0 $(($NUM_GPUS - 1))); do
37
DEVICE_NUM=$(($node * $NUM_GPUS + $node_gpu))
38
GPU_DEVICES+=$(($DEVICE_NUM))
39
- if [ $node_gpu -lt $(($NUM_GPUS - 1)) ]; then
39
+ if [ "$node_gpu" -lt $(($NUM_GPUS - 1)) ]; then
40
GPU_DEVICES+=','
41
fi
42
done
@@ -49,17 +49,20 @@ start_nodes() {
49
# 3. map the huggingface cache directory to the container
50
# 3. assign ip addresses to the containers (head node: 192.168.10.10, worker nodes:
51
# starting from 192.168.10.11)
52
- docker run -d --gpus "$GPU_DEVICES" --shm-size=10.24gb -e HF_TOKEN -v ~/.cache/huggingface:/root/.cache/huggingface --name node$node --network docker-net --ip 192.168.10.$((10 + $node)) --rm $DOCKER_IMAGE /bin/bash -c "tail -f /dev/null"
52
+ docker run -d --gpus "$GPU_DEVICES" --shm-size=10.24gb -e HF_TOKEN \
53
+ -v ~/.cache/huggingface:/root/.cache/huggingface --name "node$node" \
54
+ --network docker-net --ip 192.168.10.$((10 + $node)) --rm "$DOCKER_IMAGE" \
55
+ /bin/bash -c "tail -f /dev/null"
56
57
# organize containers into a ray cluster
55
- if [ $node -eq 0 ]; then
58
+ if [ "$node" -eq 0 ]; then
59
# start the ray head node
57
- docker exec -d node$node /bin/bash -c "ray start --head --port=6379 --block"
60
+ docker exec -d "node$node" /bin/bash -c "ray start --head --port=6379 --block"
61
# wait for the head node to be ready
62
sleep 10
63
else
64
# start the ray worker nodes, and connect them to the head node
62
- docker exec -d node$node /bin/bash -c "ray start --address=192.168.10.10:6379 --block"
65
+ docker exec -d "node$node" /bin/bash -c "ray start --address=192.168.10.10:6379 --block"
66
fi
67
done
68
@@ -79,22 +82,22 @@ run_nodes() {
82
for node_gpu in $(seq 0 $(($NUM_GPUS - 1))); do
83
DEVICE_NUM=$(($node * $NUM_GPUS + $node_gpu))
84
GPU_DEVICES+=$(($DEVICE_NUM))
82
- if [ $node_gpu -lt $(($NUM_GPUS - 1)) ]; then
85
+ if [ "$node_gpu" -lt $(($NUM_GPUS - 1)) ]; then
86
GPU_DEVICES+=','
87
fi
88
done
89
GPU_DEVICES+='"'
90
echo "Running node$node with GPU devices: $GPU_DEVICES"
88
- if [ $node -ne 0 ]; then
89
- docker exec -d node$node /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
91
+ if [ "$node" -ne 0 ]; then
92
+ docker exec -d "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
93
else
91
- docker exec node$node /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
94
+ docker exec "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
95
fi
96
done
97
}
98
cleanup() {
99
for node in $(seq 0 $(($NUM_NODES-1))); do
97
- docker stop node$node
100
+ docker stop "node$node"
101
done
102
docker network rm docker-net
103
}
.buildkite/run-neuron-test.sh
+5
-3
@@ -1,3 +1,5 @@
1
+#!/bin/bash
2
+
3
# This script build the Neuron docker image and run the API server inside the container.
4
# It serves a sanity check for compilation and basic model usage.
5
set -e
@@ -12,10 +14,10 @@ if [ -f /tmp/neuron-docker-build-timestamp ]; then
14
current_time=$(date +%s)
15
if [ $((current_time - last_build)) -gt 86400 ]; then
16
docker system prune -f
15
- echo $current_time > /tmp/neuron-docker-build-timestamp
17
+ echo "$current_time" > /tmp/neuron-docker-build-timestamp
18
fi
19
else
18
- echo $(date +%s) > /tmp/neuron-docker-build-timestamp
20
+ date "+%s" > /tmp/neuron-docker-build-timestamp
21
fi
22
23
docker build -t neuron -f Dockerfile.neuron .
@@ -34,7 +36,7 @@ wait_for_server_to_start() {
36
timeout=300
37
counter=0
38
37
- while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8000/health)" != "200" ]; do
39
+ while [ "$(curl -s -o /dev/null -w '%{http_code}' localhost:8000/health)" != "200" ]; do
40
sleep 1
41
counter=$((counter + 1))
42
if [ $counter -ge $timeout ]; then
.buildkite/run-openvino-test.sh
+2
@@ -1,3 +1,5 @@
1
+#!/bin/bash
2
+
3
# This script build the OpenVINO docker image and run the offline inference inside the container.
4
# It serves a sanity check for compilation and basic model usage.
5
set -ex
.buildkite/run-tpu-test.sh
+3
-1
@@ -1,3 +1,5 @@
1
+#!/bin/bash
2
+
3
set -e
4
5
# Build the docker image.
@@ -12,4 +14,4 @@ remove_docker_container
14
# For HF_TOKEN.
15
source /etc/environment
16
# Run a simple end-to-end example.
15
-docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && python3 -m pip install lm_eval[api]==0.4.4 && pytest -v -s /workspace/vllm/tests/entrypoints/openai/test_accuracy.py && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
17
+docker run --privileged --net host --shm-size=16G -it -e "HF_TOKEN=$HF_TOKEN" --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && python3 -m pip install lm_eval[api]==0.4.4 && pytest -v -s /workspace/vllm/tests/entrypoints/openai/test_accuracy.py && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
.buildkite/run-xpu-test.sh
+2
@@ -1,3 +1,5 @@
1
+#!/bin/bash
2
+
3
# This script build the CPU docker image and run the offline inference inside the container.
4
# It serves a sanity check for compilation and basic model usage.
5
set -ex
.github/workflows/scripts/cuda-install.sh
+4
-4
@@ -1,16 +1,16 @@
1
#!/bin/bash
2
3
# Replace '.' with '-' ex: 11.8 -> 11-8
4
-cuda_version=$(echo $1 | tr "." "-")
4
+cuda_version=$(echo "$1" | tr "." "-")
5
# Removes '-' and '.' ex: ubuntu-20.04 -> ubuntu2004
6
-OS=$(echo $2 | tr -d ".\-")
6
+OS=$(echo "$2" | tr -d ".\-")
7
8
# Installs CUDA
9
-wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-keyring_1.1-1_all.deb
9
+wget -nv "https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-keyring_1.1-1_all.deb"
10
sudo dpkg -i cuda-keyring_1.1-1_all.deb
11
rm cuda-keyring_1.1-1_all.deb
12
sudo apt -qq update
13
-sudo apt -y install cuda-${cuda_version} cuda-nvcc-${cuda_version} cuda-libraries-dev-${cuda_version}
13
+sudo apt -y install "cuda-${cuda_version}" "cuda-nvcc-${cuda_version}" "cuda-libraries-dev-${cuda_version}"
14
sudo apt clean
15
16
# Test nvcc
.github/workflows/scripts/pytorch-install.sh
+1
-1
@@ -6,7 +6,7 @@ cuda_version=$3
6
7
# Install torch
8
$python_executable -m pip install numpy pyyaml scipy ipython mkl mkl-include ninja cython typing pandas typing-extensions dataclasses setuptools && conda clean -ya
9
-$python_executable -m pip install torch==${pytorch_version}+cu${cuda_version//./} --extra-index-url https://download.pytorch.org/whl/cu${cuda_version//./}
9
+$python_executable -m pip install torch=="${pytorch_version}+cu${cuda_version//./}" --extra-index-url "https://download.pytorch.org/whl/cu${cuda_version//./}"
10
11
# Print version information
12
$python_executable --version
.github/workflows/shellcheck.yml
new
+37
@@ -0,0 +1,37 @@
1
+name: Lint shell scripts
2
+on:
3
+ push:
4
+ branches:
5
+ - "main"
6
+ paths:
7
+ - '**/*.sh'
8
+ - '.github/workflows/shellcheck.yml'
9
+ pull_request:
10
+ branches:
11
+ - "main"
12
+ paths:
13
+ - '**/*.sh'
14
+ - '.github/workflows/shellcheck.yml'
15
+
16
+env:
17
+ LC_ALL: en_US.UTF-8
18
+
19
+defaults:
20
+ run:
21
+ shell: bash
22
+
23
+permissions:
24
+ contents: read
25
+
26
+jobs:
27
+ shellcheck:
28
+ runs-on: ubuntu-latest
29
+ steps:
30
+ - name: "Checkout"
31
+ uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
32
+ with:
33
+ fetch-depth: 0
34
+
35
+ - name: "Check shell scripts"
36
+ run: |
37
+ tools/shellcheck.sh
.gitignore
+1
@@ -202,3 +202,4 @@ benchmarks/*.json
202
203
# Linting
204
actionlint
205
+shellcheck*/
.shellcheckrc
new
+9
@@ -0,0 +1,9 @@
1
+# rules currently disabled:
2
+#
3
+# SC1091 (info): Not following: <sourced file> was not specified as input (see shellcheck -x)
4
+# SC2004 (style): $/${} is unnecessary on arithmetic variables.
5
+# SC2129 (style): Consider using { cmd1; cmd2; } >> file instead of individual redirects.
6
+# SC2155 (warning): Declare and assign separately to avoid masking return values.
7
+# SC2164 (warning): Use 'cd ... || exit' or 'cd ... || return' in case cd fails.
8
+#
9
+disable=SC1091,SC2004,SC2129,SC2155,SC2164
benchmarks/launch_tgi_server.sh
+4
-4
@@ -4,13 +4,13 @@ PORT=8000
4
MODEL=$1
5
TOKENS=$2
6
7
-docker run -e HF_TOKEN=$HF_TOKEN --gpus all --shm-size 1g -p $PORT:80 \
8
- -v $PWD/data:/data \
7
+docker run -e "HF_TOKEN=$HF_TOKEN" --gpus all --shm-size 1g -p $PORT:80 \
8
+ -v "$PWD/data:/data" \
9
ghcr.io/huggingface/text-generation-inference:2.2.0 \
10
- --model-id $MODEL \
10
+ --model-id "$MODEL" \
11
--sharded false \
12
--max-input-length 1024 \
13
--max-total-tokens 2048 \
14
--max-best-of 5 \
15
--max-concurrent-requests 5000 \
16
- --max-batch-total-tokens $TOKENS
16
+ --max-batch-total-tokens "$TOKENS"
examples/run_cluster.sh
+2
-2
@@ -14,7 +14,7 @@ PATH_TO_HF_HOME="$4"
14
shift 4
15
16
# Additional arguments are passed directly to the Docker command
17
-ADDITIONAL_ARGS="$@"
17
+ADDITIONAL_ARGS=("$@")
18
19
# Validate node type
20
if [ "${NODE_TYPE}" != "--head" ] && [ "${NODE_TYPE}" != "--worker" ]; then
@@ -45,5 +45,5 @@ docker run \
45
--shm-size 10.24g \
46
--gpus all \
47
-v "${PATH_TO_HF_HOME}:/root/.cache/huggingface" \
48
- ${ADDITIONAL_ARGS} \
48
+ "${ADDITIONAL_ARGS[@]}" \
49
"${DOCKER_IMAGE}" -c "${RAY_START_CMD}"
format.sh
+7
-3
@@ -44,14 +44,14 @@ CLANGFORMAT_VERSION=$(clang-format --version | awk '{print $3}')
44
45
# # params: tool name, tool version, required version
46
tool_version_check() {
47
- if [[ $2 != $3 ]]; then
47
+ if [[ "$2" != "$3" ]]; then
48
echo "❓❓Wrong $1 version installed: $3 is required, not $2."
49
exit 1
50
fi
51
}
52
53
-tool_version_check "yapf" $YAPF_VERSION "$(grep yapf requirements-lint.txt | cut -d'=' -f3)"
54
-tool_version_check "ruff" $RUFF_VERSION "$(grep "ruff==" requirements-lint.txt | cut -d'=' -f3)"
53
+tool_version_check "yapf" "$YAPF_VERSION" "$(grep yapf requirements-lint.txt | cut -d'=' -f3)"
54
+tool_version_check "ruff" "$RUFF_VERSION" "$(grep "ruff==" requirements-lint.txt | cut -d'=' -f3)"
55
tool_version_check "mypy" "$MYPY_VERSION" "$(grep mypy requirements-lint.txt | cut -d'=' -f3)"
56
tool_version_check "isort" "$ISORT_VERSION" "$(grep isort requirements-lint.txt | cut -d'=' -f3)"
57
tool_version_check "codespell" "$CODESPELL_VERSION" "$(grep codespell requirements-lint.txt | cut -d'=' -f3)"
@@ -294,6 +294,10 @@ echo 'vLLM actionlint:'
294
tools/actionlint.sh -color
295
echo 'vLLM actionlint: Done'
296
297
+echo 'vLLM shellcheck:'
298
+tools/shellcheck.sh
299
+echo 'vLLM shellcheck: Done'
300
+
301
if ! git diff --quiet &>/dev/null; then
302
echo
303
echo "🔍🔍There are files changed by the format checker or by you that are not added and committed:"
tests/weight_loading/run_model_weight_loading_test.sh
+1
-1
@@ -14,7 +14,7 @@ while getopts "c:" OPT; do
14
done
15
16
17
-IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < $CONFIG
17
+IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "$CONFIG"
18
19
for MODEL_CONFIG in "${MODEL_CONFIGS[@]}"
20
do
tools/mypy.sh
+2
-2
@@ -3,13 +3,13 @@
3
CI=${1:-0}
4
PYTHON_VERSION=${2:-3.9}
5
6
-if [ $CI -eq 1 ]; then
6
+if [ "$CI" -eq 1 ]; then
7
set -e
8
fi
9
10
run_mypy() {
11
echo "Running mypy on $1"
12
- if [ $CI -eq 1 ] && [ -z "$1" ]; then
12
+ if [ "$CI" -eq 1 ] && [ -z "$1" ]; then
13
mypy --python-version "${PYTHON_VERSION}" "$@"
14
return
15
fi
tools/shellcheck.sh
new
+21
@@ -0,0 +1,21 @@
1
+#!/bin/bash
2
+
3
+scversion="stable"
4
+
5
+if [ -d "shellcheck-${scversion}" ]; then
6
+ export PATH="$PATH:$(pwd)/shellcheck-${scversion}"
7
+fi
8
+
9
+if ! [ -x "$(command -v shellcheck)" ]; then
10
+ if [ "$(uname -s)" != "Linux" ] || [ "$(uname -m)" != "x86_64" ]; then
11
+ echo "Please install shellcheck: https://github.com/koalaman/shellcheck?tab=readme-ov-file#installing"
12
+ exit 1
13
+ fi
14
+
15
+ # automatic local install if linux x86_64
16
+ wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${scversion?}/shellcheck-${scversion?}.linux.x86_64.tar.xz" | tar -xJv
17
+ export PATH="$PATH:$(pwd)/shellcheck-${scversion}"
18
+fi
19
+
20
+# TODO - fix warnings in .buildkite/run-amd-test.sh
21
+find . -name "*.sh" -not -path "./.deps/*" -not -path "./.buildkite/run-amd-test.sh" -exec shellcheck {} +