@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c61a98f529
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 05:00 UTC

[CI][BugFix] ShellCheck cleanup to remove baseline and preserve runtime behavior (#34514)

Signed-off-by: junuxyz <216036880+junuxyz@users.noreply.github.com>

junuxyz committed Feb 17, 2026 at 21:22 UTC c61a98f52993550e42d1bded121341fb9afb2ddf
55 files changed +338 -464
.buildkite/image_build/image_build_cpu.sh
+5 -5
@@ -11,10 +11,10 @@ REPO=$2
11 BUILDKITE_COMMIT=$3
12
13 # authenticate with AWS ECR
14 -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin $REGISTRY
14 +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
15
16 # skip build if image already exists
17 -if [[ -z $(docker manifest inspect $REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu) ]]; then
17 +if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu) ]]; then
18 echo "Image not found, proceeding with build..."
19 else
20 echo "Image found"
@@ -24,13 +24,13 @@ fi
24 # build
25 docker build --file docker/Dockerfile.cpu \
26 --build-arg max_jobs=16 \
27 - --build-arg buildkite_commit=$BUILDKITE_COMMIT \
27 + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
28 --build-arg VLLM_CPU_AVX512BF16=true \
29 --build-arg VLLM_CPU_AVX512VNNI=true \
30 --build-arg VLLM_CPU_AMXBF16=true \
31 - --tag $REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu \
31 + --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu \
32 --target vllm-test \
33 --progress plain .
34
35 # push
36 -docker push $REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu
36 +docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu
.buildkite/image_build/image_build_cpu_arm64.sh
+5 -5
@@ -11,10 +11,10 @@ REPO=$2
11 BUILDKITE_COMMIT=$3
12
13 # authenticate with AWS ECR
14 -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin $REGISTRY
14 +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
15
16 # skip build if image already exists
17 -if [[ -z $(docker manifest inspect $REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu) ]]; then
17 +if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu) ]]; then
18 echo "Image not found, proceeding with build..."
19 else
20 echo "Image found"
@@ -24,10 +24,10 @@ fi
24 # build
25 docker build --file docker/Dockerfile.cpu \
26 --build-arg max_jobs=16 \
27 - --build-arg buildkite_commit=$BUILDKITE_COMMIT \
28 - --tag $REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu \
27 + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
28 + --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu \
29 --target vllm-test \
30 --progress plain .
31
32 # push
33 -docker push $REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu
33 +docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu
.buildkite/image_build/image_build_hpu.sh
+5 -5
@@ -11,10 +11,10 @@ REPO=$2
11 BUILDKITE_COMMIT=$3
12
13 # authenticate with AWS ECR
14 -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin $REGISTRY
14 +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
15
16 # skip build if image already exists
17 -if [[ -z $(docker manifest inspect $REGISTRY/$REPO:$BUILDKITE_COMMIT-hpu) ]]; then
17 +if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu) ]]; then
18 echo "Image not found, proceeding with build..."
19 else
20 echo "Image found"
@@ -25,10 +25,10 @@ fi
25 docker build \
26 --file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \
27 --build-arg max_jobs=16 \
28 - --build-arg buildkite_commit=$BUILDKITE_COMMIT \
29 - --tag $REGISTRY/$REPO:$BUILDKITE_COMMIT-hpu \
28 + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
29 + --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu \
30 --progress plain \
31 https://github.com/vllm-project/vllm-gaudi.git
32
33 # push
34 -docker push $REGISTRY/$REPO:$BUILDKITE_COMMIT-hpu
34 +docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu
.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh
+1 -1
@@ -41,4 +41,4 @@ lm_eval --model vllm-vlm \
41 --tasks chartqa \
42 --batch_size auto \
43 --apply_chat_template \
44 - --limit $LIMIT
44 + --limit "$LIMIT"
.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh
+1 -4
@@ -20,14 +20,11 @@ usage() {
20 echo
21 }
22
23 -while getopts "m:b:l:f:t:" OPT; do
23 +while getopts "m:l:f:t:" OPT; do
24 case ${OPT} in
25 m )
26 MODEL="$OPTARG"
27 ;;
28 - b )
29 - BATCH_SIZE="$OPTARG"
30 - ;;
28 l )
29 LIMIT="$OPTARG"
30 ;;
.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh
+6 -6
@@ -15,11 +15,11 @@ DTYPE_FILTER="${DTYPE_FILTER:-}"
15 check_gpus() {
16 if command -v nvidia-smi; then
17 # check the number of GPUs and GPU type.
18 - declare -g gpu_count=$(nvidia-smi --list-gpus | wc -l)
18 + declare -g gpu_count=$(nvidia-smi --list-gpus | grep -c . || true)
19 elif command -v amd-smi; then
20 - declare -g gpu_count=$(amd-smi list | grep 'GPU' | wc -l)
20 + declare -g gpu_count=$(amd-smi list | grep -c 'GPU' || true)
21 elif command -v hl-smi; then
22 - declare -g gpu_count=$(hl-smi --list | grep -i "Module ID" | wc -l)
22 + declare -g gpu_count=$(hl-smi --list | grep -ci "Module ID" || true)
23 fi
24
25 if [[ $gpu_count -gt 0 ]]; then
@@ -47,7 +47,7 @@ check_cpus() {
47 declare -g numa_count=$(lscpu | grep "NUMA node(s):" | awk '{print $3}')
48 if [[ $numa_count -gt 0 ]]; then
49 echo "NUMA found."
50 - echo $numa_count
50 + echo "$numa_count"
51 else
52 echo "Need at least 1 NUMA to run benchmarking."
53 exit 1
@@ -434,7 +434,7 @@ run_serving_tests() {
434
435 # iterate over different max_concurrency
436 for max_concurrency in $max_concurrency_list; do
437 - new_test_name=$test_name"_qps_"$qps"_concurrency_"$max_concurrency
437 + new_test_name="${test_name}_qps_${qps}_concurrency_${max_concurrency}"
438 echo " new test name $new_test_name"
439 # pass the tensor parallel size, the compilation mode, and the optimization
440 # level to the client so that they can be used on the benchmark dashboard
@@ -471,7 +471,7 @@ run_serving_tests() {
471
472 # clean up
473 if [[ "${DRY_RUN:-0}" != "1" ]]; then
474 - kill -9 $server_pid
474 + kill -9 "$server_pid"
475 kill_gpu_processes
476 fi
477 done
.buildkite/scripts/annotate-rocm-release.sh
+1 -1
@@ -25,7 +25,7 @@ S3_REGION="${AWS_DEFAULT_REGION:-us-west-2}"
25 S3_URL="http://${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com"
26
27 # Format ROCm version for path (e.g., "7.1" -> "rocm710")
28 -ROCM_VERSION_PATH="rocm$(echo ${ROCM_VERSION} | tr -d '.')"
28 +ROCM_VERSION_PATH="rocm$(echo "${ROCM_VERSION}" | tr -d '.')"
29 ROCM_PATH="rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}"
30 buildkite-agent annotate --style 'success' --context 'rocm-release-workflow' << EOF
31 ## ROCm Wheel and Docker Image Releases
.buildkite/scripts/cache-rocm-base-wheels.sh
+3 -3
@@ -83,7 +83,7 @@ case "${1:-}" in
83 exit 1
84 fi
85
86 - WHEEL_COUNT=$(ls artifacts/rocm-base-wheels/*.whl 2>/dev/null | wc -l)
86 + WHEEL_COUNT=$(find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
87 if [[ "$WHEEL_COUNT" -eq 0 ]]; then
88 echo "ERROR: No wheels found in artifacts/rocm-base-wheels/" >&2
89 exit 1
@@ -110,9 +110,9 @@ case "${1:-}" in
110
111 echo ""
112 echo "Downloaded wheels:"
113 - ls -lh artifacts/rocm-base-wheels/
113 + find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' -exec ls -lh {} \;
114
115 - WHEEL_COUNT=$(ls artifacts/rocm-base-wheels/*.whl 2>/dev/null | wc -l)
115 + WHEEL_COUNT=$(find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
116 echo ""
117 echo "Total: $WHEEL_COUNT wheels"
118 echo "========================================"
.buildkite/scripts/cherry-pick-from-milestone.sh
+1 -1
@@ -134,7 +134,7 @@ log_info "Fetching merged PRs from milestone '${MILESTONE}'..."
134
135 # Store PR data in a temp file
136 PR_DATA=$(mktemp)
137 -trap "rm -f $PR_DATA" EXIT
137 +trap 'rm -f "$PR_DATA"' EXIT
138
139 if ! gh pr list --state merged --search "milestone:${MILESTONE}" \
140 --limit 1000 \
.buildkite/scripts/hardware_ci/run-cpu-test-ppc64le.sh
+2 -2
@@ -27,7 +27,7 @@ function cpu_tests() {
27 podman exec -it "$container_id" bash -c "
28 export TORCH_COMPILE_DISABLE=1
29 set -xve
30 - python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m" >> $HOME/test_basic.log
30 + python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m" >> "$HOME"/test_basic.log
31
32 # Run basic model test
33 podman exec -it "$container_id" bash -c "
@@ -43,7 +43,7 @@ function cpu_tests() {
43 pytest -v -s tests/models/language/generation/test_common.py::test_models[False-False-5-32-google/gemma-1.1-2b-it]
44 pytest -v -s tests/models/language/pooling/test_classification.py::test_models[float-jason9693/Qwen2.5-1.5B-apeach]
45 # TODO: Below test case tests/models/language/pooling/test_embedding.py::test_models[True-ssmits/Qwen2-7B-Instruct-embed-base] fails on ppc64le. Disabling it for time being.
46 - # pytest -v -s tests/models/language/pooling/test_embedding.py -m cpu_model" >> $HOME/test_rest.log
46 + # pytest -v -s tests/models/language/pooling/test_embedding.py -m cpu_model" >> "$HOME"/test_rest.log
47 }
48
49 # All of CPU tests are expected to be finished less than 40 mins.
.buildkite/scripts/hardware_ci/run-cpu-test.sh
+2 -2
@@ -16,5 +16,5 @@ echo "--- :docker: Building Docker image"
16 docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test -f docker/Dockerfile.cpu .
17
18 # Run the image, setting --shm-size=4g for tensor parallel.
19 -docker run --rm --cpuset-cpus=$CORE_RANGE --cpuset-mems=$NUMA_NODE -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 --shm-size=4g $IMAGE_NAME \
20 - timeout $TIMEOUT_VAL bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
19 +docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 --shm-size=4g "$IMAGE_NAME" \
20 + timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
.buildkite/scripts/hardware_ci/run-hpu-test.sh
+3 -3
@@ -7,7 +7,7 @@ set -exuo pipefail
7 # Try building the docker image
8 image_name="hpu/upstream-vllm-ci:${BUILDKITE_COMMIT}"
9 container_name="hpu-upstream-vllm-ci-${BUILDKITE_COMMIT}-container"
10 -cat <<EOF | docker build -t ${image_name} -f - .
10 +cat <<EOF | docker build -t "${image_name}" -f - .
11 FROM gaudi-base-image:latest
12
13 COPY ./ /workspace/vllm
@@ -39,12 +39,12 @@ EOF
39 # functions, while other platforms only need one remove_docker_container
40 # function.
41 EXITCODE=1
42 -remove_docker_containers() { docker rm -f ${container_name} || true; }
42 +remove_docker_containers() { docker rm -f "${container_name}" || true; }
43 trap 'remove_docker_containers; exit $EXITCODE;' EXIT
44 remove_docker_containers
45
46 echo "Running HPU plugin v1 test"
47 -docker run --rm --runtime=habana --name=${container_name} --network=host \
47 +docker run --rm --runtime=habana --name="${container_name}" --network=host \
48 -e HABANA_VISIBLE_DEVICES=all \
49 -e VLLM_SKIP_WARMUP=true \
50 -e PT_HPU_ENABLE_LAZY_COLLECTIVES=true \
.buildkite/scripts/hardware_ci/run-npu-test.sh
+15 -20
@@ -41,6 +41,7 @@ get_config() {
41 echo "Error: file '${TEST_RUN_CONFIG_FILE}' does not exist in the warehouse" >&2
42 exit 1
43 fi
44 + # shellcheck source=/dev/null
45 source "${TEST_RUN_CONFIG_FILE}"
46 echo "Base docker image name that get from configuration: ${BASE_IMAGE_NAME}"
47 return 0
@@ -48,9 +49,8 @@ get_config() {
49
50 # get test running configuration.
51 fetch_vllm_test_cfg
51 -get_config
52 # Check if the function call was successful. If not, exit the script.
53 -if [ $? -ne 0 ]; then
53 +if ! get_config; then
54 exit 1
55 fi
56
@@ -62,14 +62,14 @@ agent_idx=$(echo "${BUILDKITE_AGENT_NAME}" | awk -F'-' '{print $(NF-1)}')
62 echo "agent_idx: ${agent_idx}"
63 builder_name="cachebuilder${agent_idx}"
64 builder_cache_dir="/mnt/docker-cache${agent_idx}"
65 -mkdir -p ${builder_cache_dir}
65 +mkdir -p "${builder_cache_dir}"
66
67 # Try building the docker image
68 cat <<EOF | DOCKER_BUILDKIT=1 docker build \
69 - --add-host cache-service-vllm.nginx-pypi-cache.svc.cluster.local:${PYPI_CACHE_HOST} \
70 - --builder ${builder_name} --cache-from type=local,src=${builder_cache_dir} \
71 - --cache-to type=local,dest=${builder_cache_dir},mode=max \
72 - --progress=plain --load -t ${image_name} -f - .
69 + --add-host cache-service-vllm.nginx-pypi-cache.svc.cluster.local:"${PYPI_CACHE_HOST}" \
70 + --builder "${builder_name}" --cache-from type=local,src="${builder_cache_dir}" \
71 + --cache-to type=local,dest="${builder_cache_dir}",mode=max \
72 + --progress=plain --load -t "${image_name}" -f - .
73 FROM ${BASE_IMAGE_NAME}
74
75 # Define environments
@@ -116,7 +116,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
116 export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
117 source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
118 source /usr/local/Ascend/nnal/atb/set_env.sh && \
119 - export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
119 + export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/$(uname -i)-linux/devlib && \
120 python3 -m pip install -v -e /workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/
121
122 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
@@ -139,7 +139,7 @@ trap remove_docker_container EXIT
139 # Generate corresponding --device args based on BUILDKITE_AGENT_NAME
140 # Ascend NPU BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards, and agent_idx starts from 1.
141 # e.g. atlas-a2-001-1-2cards means this is the 1-th agent on atlas-a2-001 host, and it has 2 NPU cards.
142 -# returns --device /dev/davinci0 --device /dev/davinci1
142 +# returns one argument per line: --device, /dev/davinciX, ...
143 parse_and_gen_devices() {
144 local input="$1"
145 local index cards_num
@@ -151,29 +151,24 @@ parse_and_gen_devices() {
151 return 1
152 fi
153
154 - local devices=""
154 local i=0
155 while (( i < cards_num )); do
156 local dev_idx=$(((index - 1)*cards_num + i ))
158 - devices="$devices --device /dev/davinci${dev_idx}"
157 + printf '%s\n' "--device"
158 + printf '%s\n' "/dev/davinci${dev_idx}"
159 ((i++))
160 done
161 -
162 - # trim leading space
163 - devices="${devices#"${devices%%[![:space:]]*}"}"
164 - # Output devices: assigned to the caller variable
165 - printf '%s' "$devices"
161 }
162
168 -devices=$(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1
163 +mapfile -t device_args < <(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1
164
165 # Run the image and execute the Out-Of-Tree (OOT) platform interface test case on Ascend NPU hardware.
166 # This test checks whether the OOT platform interface is functioning properly in conjunction with
167 # the hardware plugin vllm-ascend.
168 model_cache_dir=/mnt/modelscope${agent_idx}
174 -mkdir -p ${model_cache_dir}
169 +mkdir -p "${model_cache_dir}"
170 docker run \
176 - ${devices} \
171 + "${device_args[@]}" \
172 --device /dev/davinci_manager \
173 --device /dev/devmm_svm \
174 --device /dev/hisi_hdc \
@@ -182,7 +177,7 @@ docker run \
177 -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
178 -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
179 -v /etc/ascend_install.info:/etc/ascend_install.info \
185 - -v ${model_cache_dir}:/root/.cache/modelscope \
180 + -v "${model_cache_dir}":/root/.cache/modelscope \
181 --entrypoint="" \
182 --name "${container_name}" \
183 "${image_name}" \
.buildkite/scripts/hardware_ci/run-xpu-test.sh
+1 -1
@@ -8,7 +8,7 @@ image_name="xpu/vllm-ci:${BUILDKITE_COMMIT}"
8 container_name="xpu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
9
10 # Try building the docker image
11 -docker build -t ${image_name} -f docker/Dockerfile.xpu .
11 +docker build -t "${image_name}" -f docker/Dockerfile.xpu .
12
13 # Setup cleanup
14 remove_docker_container() {
.buildkite/scripts/push-nightly-builds.sh
+10 -10
@@ -21,16 +21,16 @@ echo "Pushing original tag $ORIG_TAG_NAME$ORIG_TAG_SUFFIX to new nightly tag nam
21
22 # pull original arch-dependent images from AWS ECR Public
23 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
24 -docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:$ORIG_TAG_NAME-x86_64$ORIG_TAG_SUFFIX
25 -docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:$ORIG_TAG_NAME-aarch64$ORIG_TAG_SUFFIX
24 +docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG_NAME"-x86_64"$ORIG_TAG_SUFFIX"
25 +docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG_NAME"-aarch64"$ORIG_TAG_SUFFIX"
26 # tag arch-dependent images
27 -docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$ORIG_TAG_NAME-x86_64$ORIG_TAG_SUFFIX vllm/vllm-openai:$TAG_NAME-x86_64
28 -docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$ORIG_TAG_NAME-aarch64$ORIG_TAG_SUFFIX vllm/vllm-openai:$TAG_NAME-aarch64
27 +docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG_NAME"-x86_64"$ORIG_TAG_SUFFIX" vllm/vllm-openai:"$TAG_NAME"-x86_64
28 +docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG_NAME"-aarch64"$ORIG_TAG_SUFFIX" vllm/vllm-openai:"$TAG_NAME"-aarch64
29 # push arch-dependent images to DockerHub
30 -docker push vllm/vllm-openai:$TAG_NAME-x86_64
31 -docker push vllm/vllm-openai:$TAG_NAME-aarch64
30 +docker push vllm/vllm-openai:"$TAG_NAME"-x86_64
31 +docker push vllm/vllm-openai:"$TAG_NAME"-aarch64
32 # push arch-independent manifest to DockerHub
33 -docker manifest create vllm/vllm-openai:$TAG_NAME vllm/vllm-openai:$TAG_NAME-x86_64 vllm/vllm-openai:$TAG_NAME-aarch64 --amend
34 -docker manifest create vllm/vllm-openai:$TAG_NAME-$BUILDKITE_COMMIT vllm/vllm-openai:$TAG_NAME-x86_64 vllm/vllm-openai:$TAG_NAME-aarch64 --amend
35 -docker manifest push vllm/vllm-openai:$TAG_NAME
36 -docker manifest push vllm/vllm-openai:$TAG_NAME-$BUILDKITE_COMMIT
33 +docker manifest create vllm/vllm-openai:"$TAG_NAME" vllm/vllm-openai:"$TAG_NAME"-x86_64 vllm/vllm-openai:"$TAG_NAME"-aarch64 --amend
34 +docker manifest create vllm/vllm-openai:"$TAG_NAME"-"$BUILDKITE_COMMIT" vllm/vllm-openai:"$TAG_NAME"-x86_64 vllm/vllm-openai:"$TAG_NAME"-aarch64 --amend
35 +docker manifest push vllm/vllm-openai:"$TAG_NAME"
36 +docker manifest push vllm/vllm-openai:"$TAG_NAME"-"$BUILDKITE_COMMIT"
.buildkite/scripts/run-multi-node-test.sh
+1 -1
@@ -67,7 +67,7 @@ start_nodes() {
67 # 3. map the huggingface cache directory to the container
68 # 3. assign ip addresses to the containers (head node: 192.168.10.10, worker nodes:
69 # starting from 192.168.10.11)
70 - docker run -d $GPU_DEVICES --shm-size=10.24gb -e HF_TOKEN \
70 + docker run -d "$GPU_DEVICES" --shm-size=10.24gb -e HF_TOKEN \
71 -v ~/.cache/huggingface:/root/.cache/huggingface --name "node$node" \
72 --network docker-net --ip 192.168.10.$((10 + $node)) --rm "$DOCKER_IMAGE" \
73 /bin/bash -c "tail -f /dev/null"
.buildkite/scripts/run-prime-rl-test.sh
+1 -1
@@ -29,7 +29,7 @@ fi
29 if ! command -v uv &> /dev/null; then
30 echo "Installing UV package manager..."
31 curl -LsSf https://astral.sh/uv/install.sh | sh
32 - source $HOME/.local/bin/env
32 + source "$HOME"/.local/bin/env
33 fi
34
35 # Clone Prime-RL repository at specific branch for reproducible tests
.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh
+4 -4
@@ -51,14 +51,14 @@ for BACK in "${BACKENDS[@]}"; do
51 --enable-eplb \
52 --trust-remote-code \
53 --max-model-len 2048 \
54 - --all2all-backend $BACK \
55 - --port $PORT &
54 + --all2all-backend "$BACK" \
55 + --port "$PORT" &
56 SERVER_PID=$!
57 - wait_for_server $PORT
57 + wait_for_server "$PORT"
58
59 TAG=$(echo "$MODEL" | tr '/: \\n' '_____')
60 OUT="${OUT_DIR}/${TAG}_${BACK}.json"
61 - python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port $PORT --num-questions ${NUM_Q} --save-results ${OUT}
61 + python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port "$PORT" --num-questions "${NUM_Q}" --save-results "${OUT}"
62 python3 - <<PY
63 import json; acc=json.load(open('${OUT}'))['accuracy']
64 print(f"${MODEL} ${BACK}: accuracy {acc:.3f}")
.buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh
+6 -6
@@ -47,20 +47,20 @@ for BACK in "${BACKENDS[@]}"; do
47 vllm serve "$MODEL" \
48 --enforce-eager \
49 --enable-eplb \
50 - --all2all-backend $BACK \
50 + --all2all-backend "$BACK" \
51 --eplb-config '{"window_size":10, "step_interval":100, "num_redundant_experts":0, "log_balancedness":true}' \
52 - --tensor-parallel-size ${TENSOR_PARALLEL_SIZE} \
53 - --data-parallel-size ${DATA_PARALLEL_SIZE} \
52 + --tensor-parallel-size "${TENSOR_PARALLEL_SIZE}" \
53 + --data-parallel-size "${DATA_PARALLEL_SIZE}" \
54 --enable-expert-parallel \
55 --trust-remote-code \
56 --max-model-len 2048 \
57 - --port $PORT &
57 + --port "$PORT" &
58 SERVER_PID=$!
59 - wait_for_server $PORT
59 + wait_for_server "$PORT"
60
61 TAG=$(echo "$MODEL" | tr '/: \\n' '_____')
62 OUT="${OUT_DIR}/${TAG}_${BACK}.json"
63 - python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port $PORT --num-questions ${NUM_Q} --save-results ${OUT}
63 + python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port "$PORT" --num-questions "${NUM_Q}" --save-results "${OUT}"
64 python3 - <<PY
65 import json; acc=json.load(open('${OUT}'))['accuracy']
66 print(f"${MODEL} ${BACK}: accuracy {acc:.3f}")
.buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh
+4 -4
@@ -51,20 +51,20 @@ for BACK in "${BACKENDS[@]}"; do
51 --tensor-parallel-size 4 \
52 --enable-expert-parallel \
53 --enable-eplb \
54 - --all2all-backend $BACK \
54 + --all2all-backend "$BACK" \
55 --eplb-config '{"window_size":200,"step_interval":600,"use_async":true}' \
56 --speculative-config '{"method":"qwen3_next_mtp","num_speculative_tokens":1}' \
57 --trust-remote-code \
58 --max-model-len 2048 \
59 --gpu-memory-utilization 0.9 \
60 "${PLATFORM_ARGS[@]}" \
61 - --port $PORT &
61 + --port "$PORT" &
62 SERVER_PID=$!
63 - wait_for_server $PORT
63 + wait_for_server "$PORT"
64
65 TAG=$(echo "$MODEL" | tr '/: \\n' '_____')
66 OUT="${OUT_DIR}/${TAG}_${BACK}.json"
67 - python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port $PORT --num-questions ${NUM_Q} --save-results ${OUT}
67 + python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port "$PORT" --num-questions "${NUM_Q}" --save-results "${OUT}"
68 python3 - <<PY
69 import json; acc=json.load(open('${OUT}'))['accuracy']
70 print(f"${MODEL} ${BACK}: accuracy {acc:.3f}")
.buildkite/scripts/tpu/docker_run_bm.sh
+8 -7
@@ -9,10 +9,11 @@ ENV_FILE=$1
9
10 # For testing on local vm, use `set -a` to export all variables
11 source /etc/environment
12 -source $ENV_FILE
12 +# shellcheck source=/dev/null
13 +source "$ENV_FILE"
14
15 remove_docker_container() {
15 - docker rm -f $CONTAINER_NAME || true;
16 + docker rm -f "$CONTAINER_NAME" || true;
17 }
18
19 trap remove_docker_container EXIT
@@ -41,13 +42,13 @@ echo
42 echo "starting docker...$CONTAINER_NAME"
43 echo
44 docker run \
44 - -v $DOWNLOAD_DIR:$DOWNLOAD_DIR \
45 - --env-file $ENV_FILE \
45 + -v "$DOWNLOAD_DIR":"$DOWNLOAD_DIR" \
46 + --env-file "$ENV_FILE" \
47 -e HF_TOKEN="$HF_TOKEN" \
47 - -e TARGET_COMMIT=$BUILDKITE_COMMIT \
48 - -e MODEL=$MODEL \
48 + -e TARGET_COMMIT="$BUILDKITE_COMMIT" \
49 + -e MODEL="$MODEL" \
50 -e WORKSPACE=/workspace \
50 - --name $CONTAINER_NAME \
51 + --name "$CONTAINER_NAME" \
52 -d \
53 --privileged \
54 --network host \
.buildkite/scripts/tpu/run_bm.sh
+10 -10
@@ -42,21 +42,21 @@ echo "lanching vllm..."
42 echo "logging to $VLLM_LOG"
43 echo
44
45 -vllm serve $MODEL \
45 +vllm serve "$MODEL" \
46 --seed 42 \
47 - --max-num-seqs $MAX_NUM_SEQS \
48 - --max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \
49 - --tensor-parallel-size $TENSOR_PARALLEL_SIZE \
47 + --max-num-seqs "$MAX_NUM_SEQS" \
48 + --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" \
49 + --tensor-parallel-size "$TENSOR_PARALLEL_SIZE" \
50 --no-enable-prefix-caching \
51 - --download_dir $DOWNLOAD_DIR \
52 - --max-model-len $MAX_MODEL_LEN > "$VLLM_LOG" 2>&1 &
51 + --download_dir "$DOWNLOAD_DIR" \
52 + --max-model-len "$MAX_MODEL_LEN" > "$VLLM_LOG" 2>&1 &
53
54
55 echo "wait for 20 minutes.."
56 echo
57 # sleep 1200
58 # wait for 10 minutes...
59 -for i in {1..120}; do
59 +for _ in {1..120}; do
60 # TODO: detect other type of errors.
61 if grep -Fq "raise RuntimeError" "$VLLM_LOG"; then
62 echo "Detected RuntimeError, exiting."
@@ -78,11 +78,11 @@ echo "logging to $BM_LOG"
78 echo
79 vllm bench serve \
80 --backend vllm \
81 - --model $MODEL \
81 + --model "$MODEL" \
82 --dataset-name sonnet \
83 --dataset-path benchmarks/sonnet_4x.txt \
84 - --sonnet-input-len $INPUT_LEN \
85 - --sonnet-output-len $OUTPUT_LEN \
84 + --sonnet-input-len "$INPUT_LEN" \
85 + --sonnet-output-len "$OUTPUT_LEN" \
86 --ignore-eos > "$BM_LOG"
87
88 echo "completed..."
.buildkite/scripts/upload-nightly-wheels.sh
+6 -7
@@ -76,16 +76,15 @@ mkdir -p "$INDICES_OUTPUT_DIR"
76 # this indices have relative paths that could work as long as it is next to the wheel directory in s3
77 # i.e., the wheels are always in s3://vllm-wheels/<commit>/
78 # and indices can be placed in /<commit>/, or /nightly/, or /<version>/
79 -if [[ ! -z "$DEFAULT_VARIANT_ALIAS" ]]; then
80 - alias_arg="--alias-to-default $DEFAULT_VARIANT_ALIAS"
81 -else
82 - alias_arg=""
79 +alias_args=()
80 +if [[ -n "$DEFAULT_VARIANT_ALIAS" ]]; then
81 + alias_args=(--alias-to-default "$DEFAULT_VARIANT_ALIAS")
82 fi
83
84 # HACK: we do not need regex module here, but it is required by pre-commit hook
85 # To avoid any external dependency, we simply replace it back to the stdlib re module
86 sed -i 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py
88 -$PYTHON .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" --comment "commit $BUILDKITE_COMMIT" $alias_arg
87 +$PYTHON .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" --comment "commit $BUILDKITE_COMMIT" "${alias_args[@]}"
88
89 # copy indices to /<commit>/ unconditionally
90 echo "Uploading indices to $S3_COMMIT_PREFIX"
@@ -100,9 +99,9 @@ fi
99 # re-generate and copy to /<pure_version>/ only if it does not have "dev" in the version
100 if [[ "$version" != *"dev"* ]]; then
101 echo "Re-generating indices for /$pure_version/"
103 - rm -rf "$INDICES_OUTPUT_DIR/*"
102 + rm -rf "${INDICES_OUTPUT_DIR:?}/*"
103 mkdir -p "$INDICES_OUTPUT_DIR"
104 # wheel-dir is overridden to be the commit directory, so that the indices point to the correct wheel path
106 - $PYTHON .buildkite/scripts/generate-nightly-index.py --version "$pure_version" --wheel-dir "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" --comment "version $pure_version" $alias_arg
105 + $PYTHON .buildkite/scripts/generate-nightly-index.py --version "$pure_version" --wheel-dir "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" --comment "version $pure_version" "${alias_args[@]}"
106 aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/$pure_version/"
107 fi
.buildkite/scripts/upload-release-wheels-pypi.sh
+4 -4
@@ -7,7 +7,7 @@ SUBPATH=$BUILDKITE_COMMIT
7 S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
8
9 RELEASE_VERSION=$(buildkite-agent meta-data get release-version)
10 -GIT_VERSION=$(git describe --exact-match --tags $BUILDKITE_COMMIT 2>/dev/null)
10 +GIT_VERSION=$(git describe --exact-match --tags "$BUILDKITE_COMMIT" 2>/dev/null)
11
12 echo "Release version from Buildkite: $RELEASE_VERSION"
13
@@ -55,7 +55,7 @@ mkdir -p $DIST_DIR
55 aws s3 cp --recursive --exclude "*" --include "vllm-${PURE_VERSION}*.whl" --exclude "*dev*" --exclude "*rc[0-9]*" "$S3_COMMIT_PREFIX" $DIST_DIR
56 echo "Wheels copied to local directory"
57 # generate source tarball
58 -git archive --format=tar.gz --output="$DIST_DIR/vllm-${PURE_VERSION}.tar.gz" $BUILDKITE_COMMIT
58 +git archive --format=tar.gz --output="$DIST_DIR/vllm-${PURE_VERSION}.tar.gz" "$BUILDKITE_COMMIT"
59 ls -la $DIST_DIR
60
61 # upload wheels to PyPI (only default variant, i.e. files without '+' in the name)
@@ -65,6 +65,6 @@ if [[ -z "$PYPI_WHEEL_FILES" ]]; then
65 exit 1
66 fi
67
68 -python3 -m twine check $PYPI_WHEEL_FILES
69 -python3 -m twine upload --non-interactive --verbose $PYPI_WHEEL_FILES
68 +python3 -m twine check "$PYPI_WHEEL_FILES"
69 +python3 -m twine upload --non-interactive --verbose "$PYPI_WHEEL_FILES"
70 echo "Wheels uploaded to PyPI"
.buildkite/scripts/upload-rocm-wheels.sh
+2 -2
@@ -55,7 +55,7 @@ mkdir -p all-rocm-wheels
55 cp artifacts/rocm-base-wheels/*.whl all-rocm-wheels/ 2>/dev/null || true
56 cp artifacts/rocm-vllm-wheel/*.whl all-rocm-wheels/ 2>/dev/null || true
57
58 -WHEEL_COUNT=$(ls all-rocm-wheels/*.whl 2>/dev/null | wc -l)
58 +WHEEL_COUNT=$(find all-rocm-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
59 echo "Total wheels to upload: $WHEEL_COUNT"
60
61 if [ "$WHEEL_COUNT" -eq 0 ]; then
@@ -115,7 +115,7 @@ if [[ "$BUILDKITE_BRANCH" == "main" && "$BUILDKITE_PULL_REQUEST" == "false" ]] |
115 fi
116
117 # Extract version from vLLM wheel and update version-specific index
118 -VLLM_WHEEL=$(ls all-rocm-wheels/vllm*.whl 2>/dev/null | head -1)
118 +VLLM_WHEEL=$(find all-rocm-wheels -maxdepth 1 -name 'vllm*.whl' 2>/dev/null | head -1)
119 if [ -n "$VLLM_WHEEL" ]; then
120 VERSION=$(unzip -p "$VLLM_WHEEL" '**/METADATA' | grep '^Version: ' | cut -d' ' -f2)
121 echo "Version in wheel: $VERSION"
benchmarks/auto_tune/auto_tune.sh
+22 -22
@@ -46,10 +46,10 @@ echo "VLLM_LOGGING_LEVEL=$VLLM_LOGGING_LEVEL"
46 echo "RESULT_FILE=$RESULT"
47 echo "====================== AUTO TUNEPARAMETERS ===================="
48
49 -rm -rf $LOG_FOLDER
50 -rm -rf $PROFILE_PATH
51 -mkdir -p $LOG_FOLDER
52 -mkdir -p $PROFILE_PATH
49 +rm -rf "$LOG_FOLDER"
50 +rm -rf "$PROFILE_PATH"
51 +mkdir -p "$LOG_FOLDER"
52 +mkdir -p "$PROFILE_PATH"
53
54 cd "$BASE/vllm"
55
@@ -114,7 +114,7 @@ start_server() {
114
115 # wait for 10 minutes...
116 server_started=0
117 - for i in {1..60}; do
117 + for _ in {1..60}; do
118 # This line checks whether the server is still alive or not,
119 # since that we should always have permission to send signal to the server process.
120 kill -0 $server_pid 2> /dev/null || break
@@ -145,12 +145,12 @@ run_benchmark() {
145 local vllm_log="$LOG_FOLDER/vllm_log_${max_num_seqs}_${max_num_batched_tokens}.txt"
146 echo "vllm_log: $vllm_log"
147 echo
148 - rm -f $vllm_log
148 + rm -f "$vllm_log"
149 pkill -if "vllm serve" || true
150
151 echo "starting server..."
152 # Call start_server without a profile_dir to avoid profiling overhead
153 - start_server $gpu_memory_utilization $max_num_seqs $max_num_batched_tokens $vllm_log ""
153 + start_server "$gpu_memory_utilization" "$max_num_seqs" "$max_num_batched_tokens" "$vllm_log" ""
154 result=$?
155 if [[ "$result" -eq 1 ]]; then
156 echo "server failed to start. gpu_memory_utilization:$gpu_memory_utilization, max_num_seqs:$max_num_seqs, max_num_batched_tokens: $max_num_batched_tokens"
@@ -168,15 +168,15 @@ run_benchmark() {
168 # --profile flag is removed from this call
169 vllm bench serve \
170 --backend vllm \
171 - --model $MODEL \
171 + --model "$MODEL" \
172 --dataset-name random \
173 --random-input-len $adjusted_input_len \
174 - --random-output-len $OUTPUT_LEN \
174 + --random-output-len "$OUTPUT_LEN" \
175 --ignore-eos \
176 --disable-tqdm \
177 --request-rate inf \
178 --percentile-metrics ttft,tpot,itl,e2el \
179 - --goodput e2el:$MAX_LATENCY_ALLOWED_MS \
179 + --goodput e2el:"$MAX_LATENCY_ALLOWED_MS" \
180 --num-prompts 1000 \
181 --random-prefix-len $prefix_len \
182 --host "$HOSTNAME" \
@@ -195,20 +195,20 @@ run_benchmark() {
195 request_rate=$((${throughput%.*} + 1))
196 while ((request_rate > 0)); do
197 # clear prefix cache
198 - curl -X POST http://${HOSTNAME}:8004/reset_prefix_cache
198 + curl -X POST http://"${HOSTNAME}":8004/reset_prefix_cache
199 sleep 5
200 bm_log="$LOG_FOLDER/bm_log_${max_num_seqs}_${max_num_batched_tokens}_requestrate_${request_rate}.txt"
201 vllm bench serve \
202 --backend vllm \
203 - --model $MODEL \
203 + --model "$MODEL" \
204 --dataset-name random \
205 --random-input-len $adjusted_input_len \
206 - --random-output-len $OUTPUT_LEN \
206 + --random-output-len "$OUTPUT_LEN" \
207 --ignore-eos \
208 --disable-tqdm \
209 --request-rate $request_rate \
210 --percentile-metrics ttft,tpot,itl,e2el \
211 - --goodput e2el:$MAX_LATENCY_ALLOWED_MS \
211 + --goodput e2el:"$MAX_LATENCY_ALLOWED_MS" \
212 --num-prompts 100 \
213 --random-prefix-len $prefix_len \
214 --host "$HOSTNAME" \
@@ -255,7 +255,7 @@ gpu_memory_utilization=0.98
255 find_gpu_memory_utilization=0
256 while (( $(echo "$gpu_memory_utilization >= 0.9" | bc -l) )); do
257 # Pass empty string for profile_dir argument
258 - start_server $gpu_memory_utilization "${num_seqs_list[-1]}" "${num_batched_tokens_list[-1]}" "$LOG_FOLDER/vllm_log_gpu_memory_utilization_$gpu_memory_utilization.log" ""
258 + start_server "$gpu_memory_utilization" "${num_seqs_list[-1]}" "${num_batched_tokens_list[-1]}" "$LOG_FOLDER/vllm_log_gpu_memory_utilization_$gpu_memory_utilization.log" ""
259 result=$?
260 if [[ "$result" -eq 0 ]]; then
261 find_gpu_memory_utilization=1
@@ -274,7 +274,7 @@ fi
274
275 for num_seqs in "${num_seqs_list[@]}"; do
276 for num_batched_tokens in "${num_batched_tokens_list[@]}"; do
277 - run_benchmark $num_seqs $num_batched_tokens $gpu_memory_utilization
277 + run_benchmark "$num_seqs" "$num_batched_tokens" "$gpu_memory_utilization"
278 done
279 done
280 echo "finish permutations"
@@ -285,7 +285,7 @@ echo "finish permutations"
285 if (( $(echo "$best_throughput > 0" | bc -l) )); then
286 echo
287 echo "Benchmark tuning finished. Now running profiling on the best configuration found..."
288 - echo "Best config: max_num_seqs: $best_max_num_seqs, max_num_batched_tokens: $best_num_batched_tokens, throughput: $best_throughput"
288 + echo "Best config: max_num_seqs: $best_max_num_seqs, max_num_batched_tokens: $best_num_batched_tokens, throughput: $best_throughput, goodput: $best_goodput"
289 echo
290
291 vllm_log="$LOG_FOLDER/vllm_log_BEST_PROFILE.txt"
@@ -293,7 +293,7 @@ if (( $(echo "$best_throughput > 0" | bc -l) )); then
293
294 # Start server with the best params and profiling ENABLED
295 echo "Starting server for profiling..."
296 - start_server $gpu_memory_utilization $best_max_num_seqs $best_num_batched_tokens "$vllm_log" "$PROFILE_PATH"
296 + start_server "$gpu_memory_utilization" "$best_max_num_seqs" "$best_num_batched_tokens" "$vllm_log" "$PROFILE_PATH"
297
298 # Run benchmark with the best params and the --profile flag
299 echo "Running benchmark with profiling..."
@@ -301,15 +301,15 @@ if (( $(echo "$best_throughput > 0" | bc -l) )); then
301 adjusted_input_len=$(( INPUT_LEN - prefix_len ))
302 vllm bench serve \
303 --backend vllm \
304 - --model $MODEL \
304 + --model "$MODEL" \
305 --dataset-name random \
306 --random-input-len $adjusted_input_len \
307 - --random-output-len $OUTPUT_LEN \
307 + --random-output-len "$OUTPUT_LEN" \
308 --ignore-eos \
309 --disable-tqdm \
310 - --request-rate $best_request_rate \
310 + --request-rate "$best_request_rate" \
311 --percentile-metrics ttft,tpot,itl,e2el \
312 - --goodput e2el:$MAX_LATENCY_ALLOWED_MS \
312 + --goodput e2el:"$MAX_LATENCY_ALLOWED_MS" \
313 --num-prompts 100 \
314 --random-prefix-len $prefix_len \
315 --host "$HOSTNAME" \
benchmarks/auto_tune/batch_auto_tune.sh
+1 -1
@@ -64,7 +64,7 @@ for i in $(seq 0 $(($num_runs - 1))); do
64 else
65 STATUS="FAILURE"
66 ((FAILURE_COUNT++))
67 - FAILED_RUNS+=("Run #$((i+1)): $(echo $run_object | jq -c .)")
67 + FAILED_RUNS+=("Run #$((i+1)): $(echo "$run_object" | jq -c .)")
68 fi
69
70 RUN_OUTPUT=$(<"$RUN_OUTPUT_FILE")
benchmarks/run_structured_output_benchmark.sh
+16 -14
@@ -71,7 +71,7 @@ while [[ $# -gt 0 ]]; do
71 usage
72 ;;
73 *)
74 - echo "Unknown argument: $1\n"
74 + printf "Unknown argument: %s\n" "$1"
75 usage
76 ;;
77 esac
@@ -84,15 +84,17 @@ mkdir -p "$OUTPUT_DIR"
84 QPS_VALUES=(25 20 15 10 5 1)
85
86 # Common parameters
87 -COMMON_PARAMS="--backend $BACKEND \
88 - --model $MODEL \
89 - --dataset $DATASET \
90 - --structured-output-ratio $STRUCTURED_OUTPUT_RATIO \
91 - --save-results \
92 - --result-dir $OUTPUT_DIR \
93 - --output-len $MAX_NEW_TOKENS \
94 - --port $PORT \
95 - --tokenizer-mode $TOKENIZER_MODE"
87 +COMMON_PARAMS=(
88 + --backend "$BACKEND"
89 + --model "$MODEL"
90 + --dataset "$DATASET"
91 + --structured-output-ratio "$STRUCTURED_OUTPUT_RATIO"
92 + --save-results
93 + --result-dir "$OUTPUT_DIR"
94 + --output-len "$MAX_NEW_TOKENS"
95 + --port "$PORT"
96 + --tokenizer-mode "$TOKENIZER_MODE"
97 +)
98
99 echo "Starting structured output benchmark with model: $MODEL"
100 echo "Backend: $BACKEND"
@@ -109,17 +111,17 @@ for qps in "${QPS_VALUES[@]}"; do
111 GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
112
113 # Construct filename for this run
112 - FILENAME="${BACKEND}_${qps}qps_$(basename $MODEL)_${DATASET}_${GIT_HASH}.json"
114 + FILENAME="${BACKEND}_${qps}qps_$(basename "$MODEL")_${DATASET}_${GIT_HASH}_${GIT_BRANCH}.json"
115
116 NUM_PROMPTS=$(echo "$TOTAL_SECONDS * $qps" | bc)
117 NUM_PROMPTS=${NUM_PROMPTS%.*} # Remove fractional part
118 echo "Running benchmark with $NUM_PROMPTS prompts"
119
120 # Run the benchmark
119 - python "$SCRIPT_DIR/benchmark_serving_structured_output.py" $COMMON_PARAMS \
120 - --request-rate $qps \
121 + python "$SCRIPT_DIR/benchmark_serving_structured_output.py" "${COMMON_PARAMS[@]}" \
122 + --request-rate "$qps" \
123 --result-filename "$FILENAME" \
122 - --num-prompts $NUM_PROMPTS
124 + --num-prompts "$NUM_PROMPTS"
125
126 echo "Completed benchmark with QPS: $qps"
127 echo "----------------------------------------"
examples/online_serving/disaggregated_encoder/disagg_1e1p1d_example.sh
+15 -15
@@ -8,7 +8,7 @@ declare -a PIDS=()
8 ###############################################################################
9 MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}"
10 LOG_PATH="${LOG_PATH:-./logs}"
11 -mkdir -p $LOG_PATH
11 +mkdir -p "$LOG_PATH"
12
13 ENCODE_PORT="${ENCODE_PORT:-19534}"
14 PREFILL_PORT="${PREFILL_PORT:-19535}"
@@ -84,10 +84,10 @@ trap cleanup TERM
84
85 # clear previous cache
86 echo "remove previous ec cache folder"
87 -rm -rf $EC_SHARED_STORAGE_PATH
87 +rm -rf "$EC_SHARED_STORAGE_PATH"
88
89 echo "make ec cache folder"
90 -mkdir -p $EC_SHARED_STORAGE_PATH
90 +mkdir -p "$EC_SHARED_STORAGE_PATH"
91
92 ###############################################################################
93 # Encoder worker
@@ -100,7 +100,7 @@ CUDA_VISIBLE_DEVICES="$GPU_E" vllm serve "$MODEL" \
100 --no-enable-prefix-caching \
101 --max-num-batched-tokens 114688 \
102 --max-num-seqs 128 \
103 - --allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
103 + --allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
104 --ec-transfer-config '{
105 "ec_connector": "ECExampleConnector",
106 "ec_role": "ec_producer",
@@ -124,7 +124,7 @@ vllm serve "$MODEL" \
124 --enforce-eager \
125 --enable-request-id-headers \
126 --max-num-seqs 128 \
127 - --allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
127 + --allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
128 --ec-transfer-config '{
129 "ec_connector": "ECExampleConnector",
130 "ec_role": "ec_consumer",
@@ -152,7 +152,7 @@ vllm serve "$MODEL" \
152 --enforce-eager \
153 --enable-request-id-headers \
154 --max-num-seqs 128 \
155 - --allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
155 + --allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
156 --kv-transfer-config '{
157 "kv_connector": "NixlConnector",
158 "kv_role": "kv_consumer"
@@ -162,9 +162,9 @@ vllm serve "$MODEL" \
162 PIDS+=($!)
163
164 # Wait for workers
165 -wait_for_server $ENCODE_PORT
166 -wait_for_server $PREFILL_PORT
167 -wait_for_server $DECODE_PORT
165 +wait_for_server "$ENCODE_PORT"
166 +wait_for_server "$PREFILL_PORT"
167 +wait_for_server "$DECODE_PORT"
168
169 ###############################################################################
170 # Proxy
@@ -179,7 +179,7 @@ python disagg_epd_proxy.py \
179
180 PIDS+=($!)
181
182 -wait_for_server $PROXY_PORT
182 +wait_for_server "$PROXY_PORT"
183 echo "All services are up!"
184
185 ###############################################################################
@@ -187,14 +187,14 @@ echo "All services are up!"
187 ###############################################################################
188 echo "Running benchmark (stream)..."
189 vllm bench serve \
190 - --model $MODEL \
190 + --model "$MODEL" \
191 --backend openai-chat \
192 --endpoint /v1/chat/completions \
193 --dataset-name hf \
194 --dataset-path lmarena-ai/VisionArena-Chat \
195 --seed 0 \
196 - --num-prompts $NUM_PROMPTS \
197 - --port $PROXY_PORT
196 + --num-prompts "$NUM_PROMPTS" \
197 + --port "$PROXY_PORT"
198
199 PIDS+=($!)
200
@@ -202,10 +202,10 @@ PIDS+=($!)
202 # Single request with local image
203 ###############################################################################
204 echo "Running single request with local image (non-stream)..."
205 -curl http://127.0.0.1:${PROXY_PORT}/v1/chat/completions \
205 +curl http://127.0.0.1:"${PROXY_PORT}"/v1/chat/completions \
206 -H "Content-Type: application/json" \
207 -d '{
208 - "model": "'${MODEL}'",
208 + "model": "'"${MODEL}"'",
209 "messages": [
210 {"role": "system", "content": "You are a helpful assistant."},
211 {"role": "user", "content": [
examples/online_serving/disaggregated_encoder/disagg_1e1pd_example.sh
+13 -13
@@ -8,7 +8,7 @@ declare -a PIDS=()
8 ###############################################################################
9 MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}"
10 LOG_PATH="${LOG_PATH:-./logs}"
11 -mkdir -p $LOG_PATH
11 +mkdir -p "$LOG_PATH"
12
13 ENCODE_PORT="${ENCODE_PORT:-19534}"
14 PREFILL_DECODE_PORT="${PREFILL_DECODE_PORT:-19535}"
@@ -78,10 +78,10 @@ trap cleanup TERM
78
79 # clear previous cache
80 echo "remove previous ec cache folder"
81 -rm -rf $EC_SHARED_STORAGE_PATH
81 +rm -rf "$EC_SHARED_STORAGE_PATH"
82
83 echo "make ec cache folder"
84 -mkdir -p $EC_SHARED_STORAGE_PATH
84 +mkdir -p "$EC_SHARED_STORAGE_PATH"
85
86 ###############################################################################
87 # Encoder worker
@@ -94,7 +94,7 @@ CUDA_VISIBLE_DEVICES="$GPU_E" vllm serve "$MODEL" \
94 --no-enable-prefix-caching \
95 --max-num-batched-tokens 114688 \
96 --max-num-seqs 128 \
97 - --allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
97 + --allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
98 --ec-transfer-config '{
99 "ec_connector": "ECExampleConnector",
100 "ec_role": "ec_producer",
@@ -115,7 +115,7 @@ CUDA_VISIBLE_DEVICES="$GPU_PD" vllm serve "$MODEL" \
115 --enforce-eager \
116 --enable-request-id-headers \
117 --max-num-seqs 128 \
118 - --allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
118 + --allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
119 --ec-transfer-config '{
120 "ec_connector": "ECExampleConnector",
121 "ec_role": "ec_consumer",
@@ -128,8 +128,8 @@ CUDA_VISIBLE_DEVICES="$GPU_PD" vllm serve "$MODEL" \
128 PIDS+=($!)
129
130 # Wait for workers
131 -wait_for_server $ENCODE_PORT
132 -wait_for_server $PREFILL_DECODE_PORT
131 +wait_for_server "$ENCODE_PORT"
132 +wait_for_server "$PREFILL_DECODE_PORT"
133
134 ###############################################################################
135 # Proxy
@@ -144,7 +144,7 @@ python disagg_epd_proxy.py \
144
145 PIDS+=($!)
146
147 -wait_for_server $PROXY_PORT
147 +wait_for_server "$PROXY_PORT"
148 echo "All services are up!"
149
150 ###############################################################################
@@ -152,14 +152,14 @@ echo "All services are up!"
152 ###############################################################################
153 echo "Running benchmark (stream)..."
154 vllm bench serve \
155 - --model $MODEL \
155 + --model "$MODEL" \
156 --backend openai-chat \
157 --endpoint /v1/chat/completions \
158 --dataset-name hf \
159 --dataset-path lmarena-ai/VisionArena-Chat \
160 --seed 0 \
161 - --num-prompts $NUM_PROMPTS \
162 - --port $PROXY_PORT
161 + --num-prompts "$NUM_PROMPTS" \
162 + --port "$PROXY_PORT"
163
164 PIDS+=($!)
165
@@ -167,10 +167,10 @@ PIDS+=($!)
167 # Single request with local image
168 ###############################################################################
169 echo "Running single request with local image (non-stream)..."
170 -curl http://127.0.0.1:${PROXY_PORT}/v1/chat/completions \
170 +curl http://127.0.0.1:"${PROXY_PORT}"/v1/chat/completions \
171 -H "Content-Type: application/json" \
172 -d '{
173 - "model": "'${MODEL}'",
173 + "model": "'"${MODEL}"'",
174 "messages": [
175 {"role": "system", "content": "You are a helpful assistant."},
176 {"role": "user", "content": [
examples/online_serving/disaggregated_prefill.sh
+2 -2
@@ -54,7 +54,7 @@ wait_for_server() {
54 # You can also adjust --kv-ip and --kv-port for distributed inference.
55
56 # prefilling instance, which is the KV producer
57 -CUDA_VISIBLE_DEVICES=0 vllm serve $MODEL_NAME \
57 +CUDA_VISIBLE_DEVICES=0 vllm serve "$MODEL_NAME" \
58 --host 0.0.0.0 \
59 --port 8100 \
60 --max-model-len 100 \
@@ -64,7 +64,7 @@ CUDA_VISIBLE_DEVICES=0 vllm serve $MODEL_NAME \
64 '{"kv_connector":"P2pNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2,"kv_buffer_size":"1e9","kv_port":"14579","kv_connector_extra_config":{"proxy_ip":"'"$VLLM_HOST_IP"'","proxy_port":"30001","http_ip":"'"$VLLM_HOST_IP"'","http_port":"8100","send_type":"PUT_ASYNC"}}' &
65
66 # decoding instance, which is the KV consumer
67 -CUDA_VISIBLE_DEVICES=1 vllm serve $MODEL_NAME \
67 +CUDA_VISIBLE_DEVICES=1 vllm serve "$MODEL_NAME" \
68 --host 0.0.0.0 \
69 --port 8200 \
70 --max-model-len 100 \
examples/online_serving/disaggregated_serving/kv_events.sh
+1 -1
@@ -34,7 +34,7 @@ wait_for_server() {
34 done" && return 0 || return 1
35 }
36
37 -vllm serve $MODEL_NAME \
37 +vllm serve "$MODEL_NAME" \
38 --port 8100 \
39 --max-model-len 100 \
40 --enforce-eager \
examples/online_serving/disaggregated_serving/mooncake_connector/run_mooncake_connector.sh
+12 -11
@@ -143,7 +143,7 @@ main() {
143 IFS=',' read -ra BOOTSTRAP_PORT_ARRAY <<< "$BOOTSTRAP_PORTS"
144 IFS=',' read -ra DECODE_PORT_ARRAY <<< "$DECODE_PORTS"
145
146 - proxy_param=""
146 + proxy_args=()
147
148 # =============================================================================
149 # Launch Prefill Servers (X Producers)
@@ -156,12 +156,12 @@ main() {
156 local bootstrap_port=${BOOTSTRAP_PORT_ARRAY[$i]}
157
158 echo " Prefill server $((i+1)): GPU $gpu_id, Port $port, Bootstrap Port $bootstrap_port"
159 - VLLM_MOONCAKE_BOOTSTRAP_PORT=$bootstrap_port CUDA_VISIBLE_DEVICES=$gpu_id vllm serve $MODEL \
160 - --port $port \
159 + VLLM_MOONCAKE_BOOTSTRAP_PORT=$bootstrap_port CUDA_VISIBLE_DEVICES=$gpu_id vllm serve "$MODEL" \
160 + --port "$port" \
161 --kv-transfer-config \
162 "{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"kv_producer\"}" > prefill$((i+1)).log 2>&1 &
163 PIDS+=($!)
164 - proxy_param="${proxy_param} --prefill http://0.0.0.0:${port} $bootstrap_port"
164 + proxy_args+=(--prefill "http://0.0.0.0:${port}" "$bootstrap_port")
165 done
166
167 # =============================================================================
@@ -174,12 +174,12 @@ main() {
174 local port=${DECODE_PORT_ARRAY[$i]}
175
176 echo " Decode server $((i+1)): GPU $gpu_id, Port $port"
177 - CUDA_VISIBLE_DEVICES=$gpu_id vllm serve $MODEL \
178 - --port $port \
177 + CUDA_VISIBLE_DEVICES=$gpu_id vllm serve "$MODEL" \
178 + --port "$port" \
179 --kv-transfer-config \
180 "{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"kv_consumer\"}" > decode$((i+1)).log 2>&1 &
181 PIDS+=($!)
182 - proxy_param="${proxy_param} --decode http://0.0.0.0:${port}"
182 + proxy_args+=(--decode "http://0.0.0.0:${port}")
183 done
184
185 # =============================================================================
@@ -187,7 +187,7 @@ main() {
187 # =============================================================================
188 echo ""
189 echo "Starting proxy server on port $PROXY_PORT..."
190 - python3 mooncake_connector_proxy.py $proxy_param --port $PROXY_PORT > proxy.log 2>&1 &
190 + python3 mooncake_connector_proxy.py "${proxy_args[@]}" --port "$PROXY_PORT" > proxy.log 2>&1 &
191 PIDS+=($!)
192
193 # =============================================================================
@@ -196,9 +196,10 @@ main() {
196 echo ""
197 echo "Waiting for all servers to start..."
198 for port in "${PREFILL_PORT_ARRAY[@]}" "${DECODE_PORT_ARRAY[@]}"; do
199 - if ! wait_for_server $port; then
199 + if ! wait_for_server "$port"; then
200 echo "Failed to start server on port $port"
201 cleanup
202 + # shellcheck disable=SC2317
203 exit 1
204 fi
205 done
@@ -209,8 +210,8 @@ main() {
210 # =============================================================================
211 # Run Benchmark
212 # =============================================================================
212 - vllm bench serve --port $PROXY_PORT --seed $(date +%s) \
213 - --backend vllm --model $MODEL \
213 + vllm bench serve --port "$PROXY_PORT" --seed "$(date +%s)" \
214 + --backend vllm --model "$MODEL" \
215 --dataset-name random --random-input-len 7500 --random-output-len 200 \
216 --num-prompts 200 --burstiness 100 --request-rate 2 | tee benchmark.log
217
examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_example_p2p_nccl_xpyd.sh
+8 -7
@@ -166,10 +166,10 @@ main() {
166 local kv_port=$((21001 + i))
167
168 echo " Prefill server $((i+1)): GPU $gpu_id, Port $port, KV Port $kv_port"
169 - CUDA_VISIBLE_DEVICES=$gpu_id vllm serve $MODEL \
169 + CUDA_VISIBLE_DEVICES=$gpu_id vllm serve "$MODEL" \
170 --enforce-eager \
171 --host 0.0.0.0 \
172 - --port $port \
172 + --port "$port" \
173 --tensor-parallel-size 1 \
174 --seed 1024 \
175 --dtype float16 \
@@ -194,10 +194,10 @@ main() {
194 local kv_port=$((22001 + i))
195
196 echo " Decode server $((i+1)): GPU $gpu_id, Port $port, KV Port $kv_port"
197 - CUDA_VISIBLE_DEVICES=$gpu_id vllm serve $MODEL \
197 + CUDA_VISIBLE_DEVICES=$gpu_id vllm serve "$MODEL" \
198 --enforce-eager \
199 --host 0.0.0.0 \
200 - --port $port \
200 + --port "$port" \
201 --tensor-parallel-size 1 \
202 --seed 1024 \
203 --dtype float16 \
@@ -217,9 +217,10 @@ main() {
217 echo ""
218 echo "Waiting for all servers to start..."
219 for port in "${PREFILL_PORT_ARRAY[@]}" "${DECODE_PORT_ARRAY[@]}"; do
220 - if ! wait_for_server $port; then
220 + if ! wait_for_server "$port"; then
221 echo "Failed to start server on port $port"
222 cleanup
223 + # shellcheck disable=SC2317
224 exit 1
225 fi
226 done
@@ -231,8 +232,8 @@ main() {
232 # Run Benchmark
233 # =============================================================================
234 cd ../../../benchmarks/
234 - vllm bench serve --port 10001 --seed $(date +%s) \
235 - --model $MODEL \
235 + vllm bench serve --port 10001 --seed "$(date +%s)" \
236 + --model "$MODEL" \
237 --dataset-name random --random-input-len 7500 --random-output-len 200 \
238 --num-prompts 200 --burstiness 100 --request-rate 2 | tee benchmark.log
239
examples/online_serving/elastic_ep/bench.sh
+5 -5
@@ -50,8 +50,8 @@ while [[ $# -gt 0 ]]; do
50 done
51
52 vllm bench serve \
53 - --model $MODEL_NAME \
54 - --host $HOST \
55 - --port $PORT \
56 - --num-prompts $NUM_PROMPTS \
57 - --request-rate $REQUEST_RATE
53 + --model "$MODEL_NAME" \
54 + --host "$HOST" \
55 + --port "$PORT" \
56 + --num-prompts "$NUM_PROMPTS" \
57 + --request-rate "$REQUEST_RATE"
examples/online_serving/elastic_ep/serve_deepseek_v2.sh
+6 -6
@@ -57,15 +57,15 @@ echo "Starting vLLM server for $MODEL_NAME with data parallel size: $DATA_PARALL
57 export RAY_DEDUP_LOGS=0
58 export VLLM_USE_DEEP_GEMM=1
59
60 -vllm serve $MODEL_NAME \
61 - --data-parallel-size $DATA_PARALLEL_SIZE \
62 - --data-parallel-size-local $DATA_PARALLEL_SIZE \
60 +vllm serve "$MODEL_NAME" \
61 + --data-parallel-size "$DATA_PARALLEL_SIZE" \
62 + --data-parallel-size-local "$DATA_PARALLEL_SIZE" \
63 --data-parallel-backend ray \
64 --enforce-eager \
65 --enable-expert-parallel \
66 --enable-eplb \
67 --all2all-backend pplx \
68 - --num-redundant-experts $REDUNDANT_EXPERTS \
68 + --num-redundant-experts "$REDUNDANT_EXPERTS" \
69 --trust-remote-code \
70 - --host $HOST \
71 - --port $PORT
70 + --host "$HOST" \
71 + --port "$PORT"
examples/online_serving/multi-node-serving.sh
+4 -5
@@ -57,8 +57,7 @@ case "$subcommand" in
57
58 # Retry until the worker node connects to the head node or the timeout expires.
59 for (( i=0; i < $ray_init_timeout; i+=5 )); do
60 - ray start --address=$ray_address:$ray_port --block "${start_params[@]}"
61 - if [ $? -eq 0 ]; then
60 + if ray start --address="$ray_address":"$ray_port" --block "${start_params[@]}"; then
61 echo "Worker: Ray runtime started with head address $ray_address:$ray_port"
62 exit 0
63 fi
@@ -95,12 +94,12 @@ case "$subcommand" in
94 fi
95
96 # Start the Ray head node.
98 - ray start --head --port=$ray_port "${start_params[@]}"
97 + ray start --head --port="$ray_port" "${start_params[@]}"
98
99 # Poll Ray until every worker node is active.
100 for (( i=0; i < $ray_init_timeout; i+=5 )); do
102 - active_nodes=`python3 -c 'import ray; ray.init(); print(sum(node["Alive"] for node in ray.nodes()))'`
103 - if [ $active_nodes -eq $ray_cluster_size ]; then
101 + active_nodes=$(python3 -c 'import ray; ray.init(); print(sum(node["Alive"] for node in ray.nodes()))')
102 + if [ "$active_nodes" -eq "$ray_cluster_size" ]; then
103 echo "All ray workers are active and the ray cluster is initialized successfully."
104 exit 0
105 fi
examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh
+7 -9
@@ -22,11 +22,10 @@ check_hf_token() {
22
23 check_num_gpus() {
24 # can you check if the number of GPUs are >=2 via nvidia-smi/rocm-smi?
25 - which rocm-smi > /dev/null 2>&1
26 - if [ $? -ne 0 ]; then
25 + if ! which rocm-smi > /dev/null 2>&1; then
26 num_gpus=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)
27 else
29 - num_gpus=$(rocm-smi --showid | grep Instinct | wc -l)
28 + num_gpus=$(rocm-smi --showid | grep -c Instinct)
29 fi
30
31 if [ "$num_gpus" -lt 2 ]; then
@@ -39,8 +38,7 @@ check_num_gpus() {
38
39 ensure_python_library_installed() {
40 echo "Checking if $1 is installed..."
42 - python3 -c "import $1" > /dev/null 2>&1
43 - if [ $? -ne 0 ]; then
41 + if ! python3 -c "import $1" > /dev/null 2>&1; then
42 if [ "$1" == "nixl" ]; then
43 echo "$1 is not installed. Please refer to https://github.com/ai-dynamo/nixl for installation."
44 else
@@ -102,12 +100,12 @@ main() {
100 bash disagg_vllm_launcher.sh prefiller \
101 > >(tee prefiller.log) 2>&1 &
102 prefiller_pid=$!
105 - PIDS+=($prefiller_pid)
103 + PIDS+=("$prefiller_pid")
104
105 bash disagg_vllm_launcher.sh decoder \
106 > >(tee decoder.log) 2>&1 &
107 decoder_pid=$!
110 - PIDS+=($decoder_pid)
108 + PIDS+=("$decoder_pid")
109
110 python3 disagg_proxy_server.py \
111 --host localhost \
@@ -118,7 +116,7 @@ main() {
116 --decoder-port 8200 \
117 > >(tee proxy.log) 2>&1 &
118 proxy_pid=$!
121 - PIDS+=($proxy_pid)
119 + PIDS+=("$proxy_pid")
120
121 wait_for_server 8100
122 wait_for_server 8200
@@ -128,7 +126,7 @@ main() {
126
127 # begin benchmark
128 cd ../../../../benchmarks/
131 - vllm bench serve --port 9000 --seed $(date +%s) \
129 + vllm bench serve --port 9000 --seed "$(date +%s)" \
130 --model meta-llama/Llama-3.1-8B-Instruct \
131 --dataset-name random --random-input-len 7500 --random-output-len 200 \
132 --num-prompts 200 --burstiness 100 --request-rate 3.6 | tee benchmark.log
examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_vllm_launcher.sh
+2 -2
@@ -34,7 +34,7 @@ if [[ $1 == "prefiller" ]]; then
34 VLLM_ENABLE_V1_MULTIPROCESSING=1 \
35 VLLM_WORKER_MULTIPROC_METHOD=spawn \
36 CUDA_VISIBLE_DEVICES=0 \
37 - vllm serve $MODEL \
37 + vllm serve "$MODEL" \
38 --port 8100 \
39 --enforce-eager \
40 --kv-transfer-config \
@@ -51,7 +51,7 @@ elif [[ $1 == "decoder" ]]; then
51 VLLM_ENABLE_V1_MULTIPROCESSING=1 \
52 VLLM_WORKER_MULTIPROC_METHOD=spawn \
53 CUDA_VISIBLE_DEVICES=1 \
54 - vllm serve $MODEL \
54 + vllm serve "$MODEL" \
55 --port 8200 \
56 --enforce-eager \
57 --kv-transfer-config \
examples/pooling/embed/openai_embedding_long_text/service.sh
+1 -1
@@ -103,7 +103,7 @@ vllm serve "$MODEL_NAME" \
103 --tensor-parallel-size "$GPU_COUNT" \
104 --enforce-eager \
105 --pooler-config "$POOLER_CONFIG" \
106 - --served-model-name ${MODEL_CODE} \
106 + --served-model-name "${MODEL_CODE}" \
107 --api-key "$API_KEY" \
108 --trust-remote-code \
109 --port "$PORT" \
tests/standalone_tests/python_only_compile.sh
+2 -2
@@ -6,7 +6,7 @@ set -e
6
7 merge_base_commit=$(git merge-base HEAD origin/main)
8 echo "INFO: current merge base commit with main: $merge_base_commit"
9 -git show --oneline -s $merge_base_commit
9 +git show --oneline -s "$merge_base_commit"
10
11 # test whether the metadata.json url is valid, retry each 3 minutes up to 5 times
12 # this avoids cumbersome error messages & manual retries in case the precompiled wheel
@@ -40,7 +40,7 @@ for i in {1..5}; do
40 fi
41 fi
42 # failure handling & retry logic
43 - if [ $i -eq 5 ]; then
43 + if [ "$i" -eq 5 ]; then
44 echo "ERROR: metadata is still not available after 5 attempts."
45 echo "ERROR: Please check whether the precompiled wheel for commit $merge_base_commit is available."
46 echo " NOTE: If $merge_base_commit is a new commit on main, maybe try again after its release pipeline finishes."
tests/v1/ec_connector/integration/run_epd_correctness_test.sh
+54 -54
@@ -24,7 +24,7 @@ MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}"
24 # Set 1 to use multimodal prompts; else to use text-only
25 USE_MM_PROMPTS="${USE_MM_PROMPTS:-1}"
26 MM_FLAG=""
27 -if [ $USE_MM_PROMPTS = "1" ]; then
27 +if [ "$USE_MM_PROMPTS" = "1" ]; then
28 MM_FLAG="--use_mm_prompts"
29 fi
30
@@ -51,7 +51,7 @@ LOG_PATH="${LOG_PATH:-/tmp}"
51 BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_baseline.txt}"
52 BASELINE_PD_FILE="${BASELINE_PD_FILE:-/tmp/vllm_epd_baseline.txt}"
53
54 -mkdir -p $LOG_PATH
54 +mkdir -p "$LOG_PATH"
55
56 # Trap the SIGINT signal (triggered by Ctrl+C)
57 trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
@@ -87,20 +87,20 @@ run_baseline() {
87 # Start baseline instance
88 echo "Starting baseline instance on GPU $GPU_SINGLE, port $PORT"
89 CUDA_VISIBLE_DEVICES="$GPU_SINGLE" vllm serve "$MODEL" \
90 - --port $PORT \
90 + --port "$PORT" \
91 --enforce-eager \
92 --gpu-memory-utilization 0.7 \
93 --max-num-seqs 128 \
94 - --allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
95 - > $LOG_PATH/baseline.log 2>&1 &
94 + --allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
95 + > "$LOG_PATH"/baseline.log 2>&1 &
96
97 local BASELINE_PID=$!
98
99 # Wait for baseline to start
100 echo "Waiting for baseline instance to start..."
101 - wait_for_server $PORT
101 + wait_for_server "$PORT"
102
103 - curl http://127.0.0.1:$PORT/v1/models
103 + curl http://127.0.0.1:"$PORT"/v1/models
104 echo ""
105
106 # Run test in baseline mode
@@ -139,14 +139,14 @@ run_epd_1e_1pd() {
139 # Start encoder instance
140 echo "Starting encoder instance on GPU $GPU_E, port $ENCODE_PORT"
141 CUDA_VISIBLE_DEVICES="$GPU_E" vllm serve "$MODEL" \
142 - --port $ENCODE_PORT \
142 + --port "$ENCODE_PORT" \
143 --enforce-eager \
144 --gpu-memory-utilization 0.01 \
145 --enable-request-id-headers \
146 --no-enable-prefix-caching \
147 --max-num-batched-tokens 114688 \
148 --max-num-seqs 128 \
149 - --allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
149 + --allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
150 --ec-transfer-config '{
151 "ec_connector": "ECExampleConnector",
152 "ec_role": "ec_producer",
@@ -154,18 +154,18 @@ run_epd_1e_1pd() {
154 "shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
155 }
156 }' \
157 - > $LOG_PATH/1e1pd_encoder.log 2>&1 &
157 + > "$LOG_PATH"/1e1pd_encoder.log 2>&1 &
158 PIDS+=($!)
159
160 # Start prefill+decode instance
161 echo "Starting PD instance on GPU $GPU_PD, port $PREFILL_DECODE_PORT"
162 CUDA_VISIBLE_DEVICES="$GPU_PD" vllm serve "$MODEL" \
163 - --port $PREFILL_DECODE_PORT \
163 + --port "$PREFILL_DECODE_PORT" \
164 --enforce-eager \
165 --gpu-memory-utilization 0.7 \
166 --enable-request-id-headers \
167 --max-num-seqs 128 \
168 - --allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
168 + --allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
169 --ec-transfer-config '{
170 "ec_connector": "ECExampleConnector",
171 "ec_role": "ec_consumer",
@@ -173,32 +173,32 @@ run_epd_1e_1pd() {
173 "shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
174 }
175 }' \
176 - > $LOG_PATH/1e1pd_pd.log 2>&1 &
176 + > "$LOG_PATH"/1e1pd_pd.log 2>&1 &
177 PIDS+=($!)
178
179 # Wait for instances to start
180 echo "Waiting for encoder instance..."
181 - wait_for_server $ENCODE_PORT
181 + wait_for_server "$ENCODE_PORT"
182 echo "Waiting for PD instance..."
183 - wait_for_server $PREFILL_DECODE_PORT
183 + wait_for_server "$PREFILL_DECODE_PORT"
184
185 # Start proxy
186 echo "Starting EPD proxy on port $PROXY_PORT"
187 python "${GIT_ROOT}/examples/online_serving/disaggregated_encoder/disagg_epd_proxy.py" \
188 --host "0.0.0.0" \
189 - --port $PROXY_PORT \
189 + --port "$PROXY_PORT" \
190 --encode-servers-urls "http://localhost:$ENCODE_PORT" \
191 --prefill-servers-urls "disable" \
192 --decode-servers-urls "http://localhost:$PREFILL_DECODE_PORT" \
193 - > $LOG_PATH/1e1pd_proxy.log 2>&1 &
193 + > "$LOG_PATH"/1e1pd_proxy.log 2>&1 &
194 PIDS+=($!)
195
196 # Wait for proxy
197 echo "Waiting for proxy..."
198 - wait_for_server $PROXY_PORT
198 + wait_for_server "$PROXY_PORT"
199
200 - curl http://127.0.0.1:$PROXY_PORT/v1/models
201 - curl http://127.0.0.1:$PROXY_PORT/health
200 + curl http://127.0.0.1:"$PROXY_PORT"/v1/models
201 + curl http://127.0.0.1:"$PROXY_PORT"/health
202 echo ""
203
204 echo "All EPD (1E+1PD) services are up!"
@@ -217,7 +217,7 @@ run_epd_1e_1pd() {
217 echo "✓✓ 1E+1PD Correctness Test finished"
218 echo "Stopping EPD (1E+1PD) instances..."
219 for pid in "${PIDS[@]}"; do
220 - kill $pid 2>/dev/null || true
220 + kill "$pid" 2>/dev/null || true
221 done
222 sleep 2
223 cleanup_instances
@@ -244,17 +244,17 @@ run_baseline_1p_1d() {
244 CUDA_VISIBLE_DEVICES="$GPU_P" \
245 VLLM_NIXL_SIDE_CHANNEL_PORT=5559 \
246 vllm serve "$MODEL" \
247 - --port $PREFILL_PORT \
247 + --port "$PREFILL_PORT" \
248 --enforce-eager \
249 --gpu-memory-utilization 0.7 \
250 --enable-request-id-headers \
251 --max-num-seqs 128 \
252 - --allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
252 + --allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
253 --kv-transfer-config '{
254 "kv_connector": "NixlConnector",
255 "kv_role": "kv_producer"
256 }' \
257 - > $LOG_PATH/1p1d_prefill.log 2>&1 &
257 + > "$LOG_PATH"/1p1d_prefill.log 2>&1 &
258 PIDS+=($!)
259
260 # Start decode instance
@@ -262,40 +262,40 @@ run_baseline_1p_1d() {
262 CUDA_VISIBLE_DEVICES="$GPU_D" \
263 VLLM_NIXL_SIDE_CHANNEL_PORT=6000 \
264 vllm serve "$MODEL" \
265 - --port $DECODE_PORT \
265 + --port "$DECODE_PORT" \
266 --enforce-eager \
267 --gpu-memory-utilization 0.7 \
268 --enable-request-id-headers \
269 --max-num-seqs 128 \
270 - --allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
270 + --allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
271 --kv-transfer-config '{
272 "kv_connector": "NixlConnector",
273 "kv_role": "kv_consumer"
274 }' \
275 - > $LOG_PATH/1p1d_decode.log 2>&1 &
275 + > "$LOG_PATH"/1p1d_decode.log 2>&1 &
276 PIDS+=($!)
277
278 # Wait for instances to start
279 echo "Waiting for prefill instance..."
280 - wait_for_server $PREFILL_PORT
280 + wait_for_server "$PREFILL_PORT"
281 echo "Waiting for decode instance..."
282 - wait_for_server $DECODE_PORT
282 + wait_for_server "$DECODE_PORT"
283
284 # Start proxy
285 echo "Starting EPD proxy on port $PROXY_PORT"
286 python "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py" \
287 --host "0.0.0.0" \
288 - --port $PROXY_PORT \
289 - --prefiller-ports $PREFILL_PORT \
290 - --decoder-ports $DECODE_PORT \
291 - > $LOG_PATH/1p1d_proxy.log 2>&1 &
288 + --port "$PROXY_PORT" \
289 + --prefiller-ports "$PREFILL_PORT" \
290 + --decoder-ports "$DECODE_PORT" \
291 + > "$LOG_PATH"/1p1d_proxy.log 2>&1 &
292 PIDS+=($!)
293
294 # Wait for proxy
295 echo "Waiting for proxy..."
296 - wait_for_server $PROXY_PORT
296 + wait_for_server "$PROXY_PORT"
297
298 - curl http://127.0.0.1:$PROXY_PORT/healthcheck
298 + curl http://127.0.0.1:"$PROXY_PORT"/healthcheck
299 echo ""
300
301 echo "All PD (1P+1D) services are up!"
@@ -313,7 +313,7 @@ run_baseline_1p_1d() {
313 # Cleanup
314 echo "Stopping PD (1P+1D) instances..."
315 for pid in "${PIDS[@]}"; do
316 - kill $pid 2>/dev/null || true
316 + kill "$pid" 2>/dev/null || true
317 done
318 sleep 2
319 cleanup_instances
@@ -339,14 +339,14 @@ run_epd_1e_1p_1d() {
339 # Start encoder instance
340 echo "Starting encoder instance on GPU $GPU_E, port $ENCODE_PORT"
341 CUDA_VISIBLE_DEVICES="$GPU_E" vllm serve "$MODEL" \
342 - --port $ENCODE_PORT \
342 + --port "$ENCODE_PORT" \
343 --enforce-eager \
344 --gpu-memory-utilization 0.01 \
345 --enable-request-id-headers \
346 --no-enable-prefix-caching \
347 --max-num-batched-tokens 114688 \
348 --max-num-seqs 128 \
349 - --allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
349 + --allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
350 --ec-transfer-config '{
351 "ec_connector": "ECExampleConnector",
352 "ec_role": "ec_producer",
@@ -354,7 +354,7 @@ run_epd_1e_1p_1d() {
354 "shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
355 }
356 }' \
357 - > $LOG_PATH/1e1p1d_encoder.log 2>&1 &
357 + > "$LOG_PATH"/1e1p1d_encoder.log 2>&1 &
358 PIDS+=($!)
359
360 # Start prefill instance
@@ -362,12 +362,12 @@ run_epd_1e_1p_1d() {
362 CUDA_VISIBLE_DEVICES="$GPU_P" \
363 VLLM_NIXL_SIDE_CHANNEL_PORT=5559 \
364 vllm serve "$MODEL" \
365 - --port $PREFILL_PORT \
365 + --port "$PREFILL_PORT" \
366 --enforce-eager \
367 --gpu-memory-utilization 0.7 \
368 --enable-request-id-headers \
369 --max-num-seqs 128 \
370 - --allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
370 + --allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
371 --ec-transfer-config '{
372 "ec_connector": "ECExampleConnector",
373 "ec_role": "ec_consumer",
@@ -379,7 +379,7 @@ run_epd_1e_1p_1d() {
379 "kv_connector": "NixlConnector",
380 "kv_role": "kv_producer"
381 }' \
382 - > $LOG_PATH/1e1p1d_prefill.log 2>&1 &
382 + > "$LOG_PATH"/1e1p1d_prefill.log 2>&1 &
383 PIDS+=($!)
384
385 # Start decode instance
@@ -387,44 +387,44 @@ run_epd_1e_1p_1d() {
387 CUDA_VISIBLE_DEVICES="$GPU_D" \
388 VLLM_NIXL_SIDE_CHANNEL_PORT=6000 \
389 vllm serve "$MODEL" \
390 - --port $DECODE_PORT \
390 + --port "$DECODE_PORT" \
391 --enforce-eager \
392 --gpu-memory-utilization 0.7 \
393 --enable-request-id-headers \
394 --max-num-seqs 128 \
395 - --allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
395 + --allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
396 --kv-transfer-config '{
397 "kv_connector": "NixlConnector",
398 "kv_role": "kv_consumer"
399 }' \
400 - > $LOG_PATH/1e1p1d_decode.log 2>&1 &
400 + > "$LOG_PATH"/1e1p1d_decode.log 2>&1 &
401 PIDS+=($!)
402
403 # Wait for instances to start
404 echo "Waiting for encoder instance..."
405 - wait_for_server $ENCODE_PORT
405 + wait_for_server "$ENCODE_PORT"
406 echo "Waiting for prefill instance..."
407 - wait_for_server $PREFILL_PORT
407 + wait_for_server "$PREFILL_PORT"
408 echo "Waiting for decode instance..."
409 - wait_for_server $DECODE_PORT
409 + wait_for_server "$DECODE_PORT"
410
411 # Start proxy
412 echo "Starting EPD proxy on port $PROXY_PORT"
413 python "${GIT_ROOT}/examples/online_serving/disaggregated_encoder/disagg_epd_proxy.py" \
414 --host "0.0.0.0" \
415 - --port $PROXY_PORT \
415 + --port "$PROXY_PORT" \
416 --encode-servers-urls "http://localhost:$ENCODE_PORT" \
417 --prefill-servers-urls "http://localhost:$PREFILL_PORT" \
418 --decode-servers-urls "http://localhost:$DECODE_PORT" \
419 - > $LOG_PATH/1e1p1d_proxy.log 2>&1 &
419 + > "$LOG_PATH"/1e1p1d_proxy.log 2>&1 &
420 PIDS+=($!)
421
422 # Wait for proxy
423 echo "Waiting for proxy..."
424 - wait_for_server $PROXY_PORT
424 + wait_for_server "$PROXY_PORT"
425
426 - curl http://127.0.0.1:$PROXY_PORT/v1/models
427 - curl http://127.0.0.1:$PROXY_PORT/health
426 + curl http://127.0.0.1:"$PROXY_PORT"/v1/models
427 + curl http://127.0.0.1:"$PROXY_PORT"/health
428 echo ""
429
430 echo "All EPD (1E+1P+1D) services are up!"
@@ -443,7 +443,7 @@ run_epd_1e_1p_1d() {
443 echo "✓✓ 1E+1P+1D Correctness Test finished"
444 echo "Stopping EPD (1E+1P+1D) instances..."
445 for pid in "${PIDS[@]}"; do
446 - kill $pid 2>/dev/null || true
446 + kill "$pid" 2>/dev/null || true
447 done
448 sleep 2
449 cleanup_instances
tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
+6 -1
@@ -32,9 +32,14 @@ run_tests() {
32
33 echo "=== Running tests (${label}) ==="
34 for cfg in "${configs[@]}"; do
35 + local -a cfg_parts extra_args_parts
36 + read -r -a cfg_parts <<< "$cfg"
37 + read -r -a extra_args_parts <<< "$extra_args"
38 +
39 echo "-> Running with ${cfg} ${extra_args:+and ${extra_args}}"
40 # Use 'env' to safely set variables without eval
37 - if ! env ${cfg} bash "${SCRIPT}" ${extra_args}; then
41 + # keep argv splitting safe and SC2086-clean via arrays.
42 + if ! env "${cfg_parts[@]}" bash "${SCRIPT}" "${extra_args_parts[@]}"; then
43 echo "❌ Test failed for config: ${cfg} ${extra_args:+(${extra_args})}"
44 exit 1
45 fi
tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh
+11 -11
@@ -109,9 +109,9 @@ get_model_args() {
109
110 get_num_gpus() {
111 if [[ "$SMI_BIN" == *"nvidia"* ]]; then
112 - echo "$($SMI_BIN --query-gpu=name --format=csv,noheader | wc -l)"
112 + $SMI_BIN --query-gpu=name --format=csv,noheader | wc -l
113 elif [[ "$SMI_BIN" == *"rocm"* ]]; then
114 - echo "$($SMI_BIN -l | grep GPU | wc -l)"
114 + $SMI_BIN -l | grep -c GPU
115 else
116 # works for non-cuda platforms,
117 # assuming at least 1 device and
@@ -182,7 +182,7 @@ run_tests_for_model() {
182
183 # Store host and port for proxy configuration
184 PREFILL_HOSTS+=("localhost")
185 - PREFILL_PORTS+=($PORT)
185 + PREFILL_PORTS+=("$PORT")
186 done
187
188 # Start decode instances
@@ -237,30 +237,30 @@ run_tests_for_model() {
237
238 # Store host and port for proxy configuration
239 DECODE_HOSTS+=("localhost")
240 - DECODE_PORTS+=($PORT)
240 + DECODE_PORTS+=("$PORT")
241 done
242
243 # Wait for all instances to start
244 for PORT in "${PREFILL_PORTS[@]}"; do
245 echo "Waiting for prefill instance on port $PORT to start..."
246 - wait_for_server $PORT
246 + wait_for_server "$PORT"
247 done
248
249 for PORT in "${DECODE_PORTS[@]}"; do
250 echo "Waiting for decode instance on port $PORT to start..."
251 - wait_for_server $PORT
251 + wait_for_server "$PORT"
252 done
253
254 # Build the command for the proxy server with all the hosts and ports
255 PROXY_CMD="python3 ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py --port 8192"
256
257 # Add all prefill hosts and ports
258 - PROXY_CMD+=" --prefiller-hosts ${PREFILL_HOSTS[@]}"
259 - PROXY_CMD+=" --prefiller-ports ${PREFILL_PORTS[@]}"
258 + PROXY_CMD+=" --prefiller-hosts ${PREFILL_HOSTS[*]}"
259 + PROXY_CMD+=" --prefiller-ports ${PREFILL_PORTS[*]}"
260
261 # Add all decode hosts and ports
262 - PROXY_CMD+=" --decoder-hosts ${DECODE_HOSTS[@]}"
263 - PROXY_CMD+=" --decoder-ports ${DECODE_PORTS[@]}"
262 + PROXY_CMD+=" --decoder-hosts ${DECODE_HOSTS[*]}"
263 + PROXY_CMD+=" --decoder-ports ${DECODE_PORTS[*]}"
264
265 # Start the proxy server
266 echo "Starting proxy server with command: $PROXY_CMD"
@@ -271,7 +271,7 @@ run_tests_for_model() {
271
272 # Run lm eval for this model
273 echo "Running tests for $model_name"
274 - TEST_MODEL=$model_name python3 -m pytest -s -x ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_accuracy.py
274 + TEST_MODEL=$model_name python3 -m pytest -s -x "${GIT_ROOT}"/tests/v1/kv_connector/nixl_integration/test_accuracy.py
275
276 # Clean up before running next model
277 cleanup_instances
tests/v1/kv_connector/nixl_integration/run_edge_case_test.sh
+5 -5
@@ -114,10 +114,10 @@ run_tests_for_model() {
114 eval "$FULL_CMD &"
115
116 # Wait for all instances to start
117 - echo "Waiting for prefill instance on port $PORT to start..."
118 - wait_for_server $PREFILL_PORT
119 - echo "Waiting for decode instance on port $PORT to start..."
120 - wait_for_server $DECODE_PORT
117 + echo "Waiting for prefill instance on port $PREFILL_PORT to start..."
118 + wait_for_server "$PREFILL_PORT"
119 + echo "Waiting for decode instance on port $DECODE_PORT to start..."
120 + wait_for_server "$DECODE_PORT"
121
122 # Build the command for the proxy server with all the hosts and ports
123 PROXY_PORT=8192
@@ -133,7 +133,7 @@ run_tests_for_model() {
133
134 # Run lm eval for this model
135 echo "Running tests for $model_name"
136 - PREFILL_PORT=$PREFILL_PORT DECODE_PORT=$DECODE_PORT PROXY_PORT=$PROXY_PORT python -m pytest -s -v ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_edge_cases.py
136 + PREFILL_PORT=$PREFILL_PORT DECODE_PORT=$DECODE_PORT PROXY_PORT=$PROXY_PORT python -m pytest -s -v "${GIT_ROOT}"/tests/v1/kv_connector/nixl_integration/test_edge_cases.py
137
138 # Clean up before running next model
139 cleanup_instances
tests/v1/kv_connector/nixl_integration/run_tpu_disagg_accuracy_test.sh
+13 -13
@@ -63,8 +63,8 @@ launch_baseline() {
63 --block-size ${BLOCK_SIZE} \
64 --gpu-memory-utilization 0.5 \
65 --enforce-eager"
66 - echo ${BASELINE_BASE_CMD}
67 - ssh -tt ${BASELINE_HOST} "${BASELINE_BASE_CMD}" &
66 + echo "${BASELINE_BASE_CMD}"
67 + ssh -tt "${BASELINE_HOST}" "${BASELINE_BASE_CMD}" &
68 }
69
70 launch_pd() {
@@ -103,17 +103,17 @@ launch_pd() {
103 --gpu-memory-utilization 0.5 \
104 --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"cpu\"}'"
105
106 - echo ${PREFILL_BASE_CMD}
107 - echo ${DECODE_BASE_CMD}
106 + echo "${PREFILL_BASE_CMD}"
107 + echo "${DECODE_BASE_CMD}"
108 sleep 2
109
110 # execute on hosts
111 - ssh -tt ${PREFILL_HOST} "${PREFILL_BASE_CMD}" &
112 - ssh -tt ${DECODE_HOST} "${DECODE_BASE_CMD}" &
111 + ssh -tt "${PREFILL_HOST}" "${PREFILL_BASE_CMD}" &
112 + ssh -tt "${DECODE_HOST}" "${DECODE_BASE_CMD}" &
113 sleep 1
114 - wait_for_server ${PREFILL_HOST} ${PREFILL_PORT}
114 + wait_for_server "${PREFILL_HOST}" "${PREFILL_PORT}"
115 sleep 1
116 - wait_for_server ${DECODE_HOST} ${DECODE_PORT}
116 + wait_for_server "${DECODE_HOST}" "${DECODE_PORT}"
117 sleep 1
118 }
119
@@ -123,21 +123,21 @@ launch_pd_proxy(){
123 --prefiller-host ${PREFILL_HOST} --prefiller-port ${PREFILL_PORT} \
124 --decoder-host ${DECODE_HOST} --decoder-port ${DECODE_PORT} \
125 --host=${PROXY_HOST} --port ${PROXY_PORT}"
126 - echo ${PROXY_BASE_CMD}
127 - ssh -tt ${PROXY_HOST} "${PROXY_BASE_CMD}" &
126 + echo "${PROXY_BASE_CMD}"
127 + ssh -tt "${PROXY_HOST}" "${PROXY_BASE_CMD}" &
128 }
129
130 run_tests(){
131 local service_url=$1
132 local mode=$2
133 - python3 ${EXP_ROOT}/test_disagg_accuracy.py --service_url=${service_url} --model_name=${MODEL_NAME} --mode=${mode} --file_name=${OUTPUT_FILE}
133 + python3 "${EXP_ROOT}"/test_disagg_accuracy.py --service_url="${service_url}" --model_name="${MODEL_NAME}" --mode="${mode}" --file_name="${OUTPUT_FILE}"
134 }
135
136
137 # run non-disagg. baseline & save outputs
138 launch_baseline
139 sleep 2
140 -wait_for_server ${BASELINE_HOST} ${BASELINE_PORT}
140 +wait_for_server "${BASELINE_HOST}" "${BASELINE_PORT}"
141 run_tests "http://${BASELINE_HOST}:${BASELINE_PORT}" "baseline"
142 cleanup
143 sleep 10
@@ -150,7 +150,7 @@ sleep 10
150 run_tests "http://${PROXY_HOST}:${PROXY_PORT}" "disagg"
151 echo "-----P/D success----"
152
153 -rm ${OUTPUT_FILE}
153 +rm "${OUTPUT_FILE}"
154 cleanup
155
156 exit 0
\ No newline at end of file
tests/v1/kv_connector/nixl_integration/run_tpu_edge_case_test.sh
+9 -9
@@ -86,17 +86,17 @@ launch_pd() {
86 --gpu-memory-utilization 0.5 \
87 --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"cpu\"}'"
88
89 - echo ${PREFILL_BASE_CMD}
90 - echo ${DECODE_BASE_CMD}
89 + echo "${PREFILL_BASE_CMD}"
90 + echo "${DECODE_BASE_CMD}"
91 sleep 2
92
93 # execute on hosts
94 - ssh -tt ${PREFILL_HOST} "${PREFILL_BASE_CMD}" &
95 - ssh -tt ${DECODE_HOST} "${DECODE_BASE_CMD}" &
94 + ssh -tt "${PREFILL_HOST}" "${PREFILL_BASE_CMD}" &
95 + ssh -tt "${DECODE_HOST}" "${DECODE_BASE_CMD}" &
96 sleep 1
97 - wait_for_server ${PREFILL_HOST} ${PREFILL_PORT}
97 + wait_for_server "${PREFILL_HOST}" "${PREFILL_PORT}"
98 sleep 1
99 - wait_for_server ${DECODE_HOST} ${DECODE_PORT}
99 + wait_for_server "${DECODE_HOST}" "${DECODE_PORT}"
100 sleep 1
101 }
102
@@ -106,8 +106,8 @@ launch_pd_proxy(){
106 --prefiller-host ${PREFILL_HOST} --prefiller-port ${PREFILL_PORT} \
107 --decoder-host ${DECODE_HOST} --decoder-port ${DECODE_PORT} \
108 --host=${PROXY_HOST} --port ${PROXY_PORT}"
109 - echo ${PROXY_BASE_CMD}
110 - ssh -tt ${PROXY_HOST} "${PROXY_BASE_CMD}" &
109 + echo "${PROXY_BASE_CMD}"
110 + ssh -tt "${PROXY_HOST}" "${PROXY_BASE_CMD}" &
111 }
112
113
@@ -121,4 +121,4 @@ PREFILL_PORT=${PREFILL_PORT} \
121 DECODE_HOST=${DECODE_HOST} \
122 DECODE_PORT=${DECODE_PORT} \
123 PROXY_HOST=${PROXY_HOST} \
124 -PROXY_PORT=${PROXY_PORT} python -m pytest -s -v ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_edge_cases.py
\ No newline at end of file
124 +PROXY_PORT=${PROXY_PORT} python -m pytest -s -v "${GIT_ROOT}"/tests/v1/kv_connector/nixl_integration/test_edge_cases.py
tools/ep_kernels/elastic_ep/install_eep_libraries.sh
+5 -5
@@ -23,7 +23,7 @@ while getopts "w:n" opt; do
23 done
24
25 if [ ! -d "$WORKSPACE" ]; then
26 - mkdir -p $WORKSPACE
26 + mkdir -p "$WORKSPACE"
27 fi
28
29
@@ -31,7 +31,7 @@ fi
31 pip3 install cmake torch ninja
32
33 # build nvshmem
34 -pushd $WORKSPACE
34 +pushd "$WORKSPACE"
35 # Reset NVSHMEM build if requested
36 if [ "$INSTALL_NVSHMEM" = true ]; then
37 mkdir -p nvshmem_src
@@ -69,15 +69,15 @@ export NVSHMEM_BUILD_HYDRA_LAUNCHER=0
69 export NVSHMEM_BUILD_TXZ_PACKAGE=0
70 export NVSHMEM_TIMEOUT_DEVICE_POLLING=0
71
72 -cmake -G Ninja -S . -B $WORKSPACE/nvshmem_build/ -DCMAKE_INSTALL_PREFIX=$WORKSPACE/nvshmem_install
73 -cmake --build $WORKSPACE/nvshmem_build/ --target install
72 +cmake -G Ninja -S . -B "$WORKSPACE"/nvshmem_build/ -DCMAKE_INSTALL_PREFIX="$WORKSPACE"/nvshmem_install
73 +cmake --build "$WORKSPACE"/nvshmem_build/ --target install
74
75 popd
76
77 export CMAKE_PREFIX_PATH=$WORKSPACE/nvshmem_install:$CMAKE_PREFIX_PATH
78
79 # build and install pplx, require pytorch installed
80 -pushd $WORKSPACE
80 +pushd "$WORKSPACE"
81 git clone https://github.com/ppl-ai/pplx-kernels
82 cd pplx-kernels
83 # see https://github.com/pypa/pip/issues/9955#issuecomment-838065925
tools/ep_kernels/install_python_libraries.sh
+1 -1
@@ -14,7 +14,7 @@ DEEPEP_COMMIT_HASH=${DEEPEP_COMMIT_HASH:-"73b6ea4"}
14 NVSHMEM_VER=${NVSHMEM_VER:-"3.3.24"} # Default supports both CUDA 12 and 13
15 WORKSPACE=${WORKSPACE:-$(pwd)/ep_kernels_workspace}
16 MODE=${MODE:-install}
17 -CUDA_VERSION_MAJOR=$(${CUDA_HOME}/bin/nvcc --version | egrep -o "release [0-9]+" | cut -d ' ' -f 2)
17 +CUDA_VERSION_MAJOR=$("${CUDA_HOME}"/bin/nvcc --version | grep -E -o "release [0-9]+" | cut -d ' ' -f 2)
18
19 # Parse arguments
20 while [[ $# -gt 0 ]]; do
tools/flashinfer-build.sh
+3 -5
@@ -5,8 +5,6 @@ set -ex
5
6 # FlashInfer configuration
7 FLASHINFER_GIT_REPO="https://github.com/flashinfer-ai/flashinfer.git"
8 -FLASHINFER_GIT_REF="${FLASHINFER_GIT_REF}"
9 -CUDA_VERSION="${CUDA_VERSION}"
8 BUILD_WHEEL="${BUILD_WHEEL:-true}"
9
10 if [[ -z "${FLASHINFER_GIT_REF}" ]]; then
@@ -23,7 +21,7 @@ echo "🏗️ Building FlashInfer ${FLASHINFER_GIT_REF} for CUDA ${CUDA_VERSION
21
22 # Clone FlashInfer
23 git clone --depth 1 --recursive --shallow-submodules \
26 - --branch ${FLASHINFER_GIT_REF} \
24 + --branch "${FLASHINFER_GIT_REF}" \
25 ${FLASHINFER_GIT_REPO} flashinfer
26
27 # Set CUDA arch list based on CUDA version
@@ -44,7 +42,7 @@ echo "🏗️ Building FlashInfer AOT for arches: ${FI_TORCH_CUDA_ARCH_LIST}"
42
43 pushd flashinfer
44 # Make sure the wheel is built for the correct CUDA version
47 - export UV_TORCH_BACKEND=cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
45 + export UV_TORCH_BACKEND=cu$(echo "$CUDA_VERSION" | cut -d. -f1,2 | tr -d '.')
46
47 # Build AOT kernels
48 export TORCH_CUDA_ARCH_LIST="${FI_TORCH_CUDA_ARCH_LIST}"
@@ -63,4 +61,4 @@ pushd flashinfer
61 popd
62
63 # Cleanup
66 -rm -rf flashinfer
\ No newline at end of file
64 +rm -rf flashinfer
tools/install_deepgemm.sh
+2 -2
@@ -65,7 +65,7 @@ fi
65
66 # Extract major and minor version numbers
67 CUDA_MAJOR="${CUDA_VERSION%%.*}"
68 -CUDA_MINOR="${CUDA_VERSION#${CUDA_MAJOR}.}"
68 +CUDA_MINOR="${CUDA_VERSION#"${CUDA_MAJOR}".}"
69 CUDA_MINOR="${CUDA_MINOR%%.*}"
70 echo "CUDA version: $CUDA_VERSION (major: $CUDA_MAJOR, minor: $CUDA_MINOR)"
71
@@ -92,7 +92,7 @@ git checkout "$DEEPGEMM_GIT_REF"
92
93 # Clean previous build artifacts
94 # (Based on https://github.com/deepseek-ai/DeepGEMM/blob/main/install.sh)
95 -rm -rf build dist *.egg-info
95 +rm -rf -- build dist *.egg-info 2>/dev/null || true
96
97 # Build wheel
98 echo "🏗️ Building DeepGEMM wheel..."
tools/pre_commit/shellcheck.baseline deleted
-89
@@ -1,89 +0,0 @@
1 -benchmarks/auto_tune/auto_tune.sh:SC2034
2 -benchmarks/auto_tune/auto_tune.sh:SC2086
3 -benchmarks/auto_tune/batch_auto_tune.sh:SC2086
4 -benchmarks/run_structured_output_benchmark.sh:SC2028
5 -benchmarks/run_structured_output_benchmark.sh:SC2034
6 -benchmarks/run_structured_output_benchmark.sh:SC2086
7 -.buildkite/image_build/image_build_cpu_arm64.sh:SC2086
8 -.buildkite/image_build/image_build_cpu.sh:SC2086
9 -.buildkite/image_build/image_build_hpu.sh:SC2086
10 -.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh:SC2086
11 -.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh:SC2034
12 -.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh:SC2027
13 -.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh:SC2086
14 -.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh:SC2126
15 -.buildkite/scripts/annotate-rocm-release.sh:SC2086
16 -.buildkite/scripts/cache-rocm-base-wheels.sh:SC2012
17 -.buildkite/scripts/cherry-pick-from-milestone.sh:SC2064
18 -.buildkite/scripts/hardware_ci/run-cpu-test-ppc64le.sh:SC2086
19 -.buildkite/scripts/hardware_ci/run-cpu-test.sh:SC2086
20 -.buildkite/scripts/hardware_ci/run-hpu-test.sh:SC2086
21 -.buildkite/scripts/hardware_ci/run-npu-test.sh:SC1090
22 -.buildkite/scripts/hardware_ci/run-npu-test.sh:SC2006
23 -.buildkite/scripts/hardware_ci/run-npu-test.sh:SC2086
24 -.buildkite/scripts/hardware_ci/run-npu-test.sh:SC2181
25 -.buildkite/scripts/hardware_ci/run-xpu-test.sh:SC2086
26 -.buildkite/scripts/push-nightly-builds.sh:SC2086
27 -.buildkite/scripts/run-multi-node-test.sh:SC2086
28 -.buildkite/scripts/run-multi-node-test.sh:SC2089
29 -.buildkite/scripts/run-multi-node-test.sh:SC2090
30 -.buildkite/scripts/run-prime-rl-test.sh:SC2086
31 -.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh:SC2086
32 -.buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh:SC2086
33 -.buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh:SC2086
34 -.buildkite/scripts/tpu/docker_run_bm.sh:SC1090
35 -.buildkite/scripts/tpu/docker_run_bm.sh:SC2086
36 -.buildkite/scripts/tpu/run_bm.sh:SC2034
37 -.buildkite/scripts/tpu/run_bm.sh:SC2086
38 -.buildkite/scripts/upload-nightly-wheels.sh:SC2086
39 -.buildkite/scripts/upload-nightly-wheels.sh:SC2115
40 -.buildkite/scripts/upload-nightly-wheels.sh:SC2236
41 -.buildkite/scripts/upload-release-wheels-pypi.sh:SC2086
42 -.buildkite/scripts/upload-rocm-wheels.sh:SC2012
43 -examples/online_serving/disaggregated_encoder/disagg_1e1p1d_example.sh:SC2086
44 -examples/online_serving/disaggregated_encoder/disagg_1e1pd_example.sh:SC2086
45 -examples/online_serving/disaggregated_prefill.sh:SC2086
46 -examples/online_serving/disaggregated_serving/kv_events.sh:SC2086
47 -examples/online_serving/disaggregated_serving/mooncake_connector/run_mooncake_connector.sh:SC2046
48 -examples/online_serving/disaggregated_serving/mooncake_connector/run_mooncake_connector.sh:SC2086
49 -examples/online_serving/disaggregated_serving/mooncake_connector/run_mooncake_connector.sh:SC2317
50 -examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_example_p2p_nccl_xpyd.sh:SC2046
51 -examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_example_p2p_nccl_xpyd.sh:SC2086
52 -examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_example_p2p_nccl_xpyd.sh:SC2317
53 -examples/online_serving/elastic_ep/bench.sh:SC2086
54 -examples/online_serving/elastic_ep/serve_deepseek_v2.sh:SC2086
55 -examples/online_serving/multi-node-serving.sh:SC2006
56 -examples/online_serving/multi-node-serving.sh:SC2086
57 -examples/online_serving/multi-node-serving.sh:SC2181
58 -examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh:SC2046
59 -examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh:SC2126
60 -examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh:SC2181
61 -examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh:SC2206
62 -examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_vllm_launcher.sh:SC2086
63 -examples/pooling/embed/openai_embedding_long_text/service.sh:SC2086
64 -tests/standalone_tests/python_only_compile.sh:SC2086
65 -tests/v1/ec_connector/integration/run_epd_correctness_test.sh:SC2086
66 -tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh:SC2086
67 -tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh:SC2005
68 -tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh:SC2086
69 -tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh:SC2124
70 -tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh:SC2126
71 -tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh:SC2206
72 -tests/v1/kv_connector/nixl_integration/run_edge_case_test.sh:SC2086
73 -tests/v1/kv_connector/nixl_integration/run_edge_case_test.sh:SC2153
74 -tests/v1/kv_connector/nixl_integration/run_tpu_disagg_accuracy_test.sh:SC2086
75 -tests/v1/kv_connector/nixl_integration/run_tpu_disagg_accuracy_test.sh:SC2089
76 -tests/v1/kv_connector/nixl_integration/run_tpu_disagg_accuracy_test.sh:SC2090
77 -tests/v1/kv_connector/nixl_integration/run_tpu_edge_case_test.sh:SC2086
78 -tests/v1/kv_connector/nixl_integration/run_tpu_edge_case_test.sh:SC2089
79 -tests/v1/kv_connector/nixl_integration/run_tpu_edge_case_test.sh:SC2090
80 -tools/ep_kernels/elastic_ep/install_eep_libraries.sh:SC2086
81 -tools/ep_kernels/install_python_libraries.sh:SC2086
82 -tools/ep_kernels/install_python_libraries.sh:SC2196
83 -tools/flashinfer-build.sh:SC2086
84 -tools/flashinfer-build.sh:SC2269
85 -tools/install_deepgemm.sh:SC2035
86 -tools/install_deepgemm.sh:SC2295
87 -tools/pre_commit/shellcheck.sh:SC2016
88 -tools/vllm-rocm/generate-rocm-wheels-root-index.sh:SC2295
89 -tools/vllm-tpu/build.sh:SC2145
tools/pre_commit/shellcheck.sh
+3 -36
@@ -2,7 +2,6 @@
2 set -euo pipefail
3
4 scversion="stable"
5 -baseline="tools/pre_commit/shellcheck.baseline"
5
6 if [ -d "shellcheck-${scversion}" ]; then
7 export PATH="$PATH:$(pwd)/shellcheck-${scversion}"
@@ -20,38 +19,6 @@ if ! [ -x "$(command -v shellcheck)" ]; then
19 fi
20
21 # TODO - fix warnings in .buildkite/scripts/hardware_ci/run-amd-test.sh
23 -# collects warnings as "file:SCcode" pairs for baseline comparison.
24 -collect() {
25 - find . -path ./.git -prune -o -name "*.sh" \
26 - -not -path "./.buildkite/scripts/hardware_ci/run-amd-test.sh" -print0 | \
27 - xargs -0 sh -c 'for f in "$@"; do git check-ignore -q "$f" || shellcheck -s bash -f gcc "$f" || true; done' -- | \
28 - sed -nE 's|^\./||; s|^([^:]+):[0-9]+:[0-9]+:.*\[(SC[0-9]+)\]$|\1:\2|p' | \
29 - sort -u
30 -}
31 -
32 -if [[ "${1:-}" == "--generate-baseline" ]]; then
33 - collect > "$baseline"
34 - echo "Wrote baseline to $baseline"
35 - exit 0
36 -fi
37 -
38 -if [[ ! -f "$baseline" ]]; then
39 - echo "Baseline not found: $baseline (run: $0 --generate-baseline)"
40 - exit 1
41 -fi
42 -
43 -current="$(mktemp)"
44 -trap 'rm -f "$current"' EXIT
45 -collect > "$current"
46 -
47 -# finds new warnings not in baseline
48 -new_errors="$(comm -23 "$current" <(sort -u "$baseline") || true)"
49 -if [ -n "$new_errors" ]; then
50 - echo "$new_errors" | cut -d: -f1 | sort -u | while IFS= read -r file; do
51 - if [[ -f "$file" ]]; then
52 - codes=$(echo "$new_errors" | awk -F: -v f="$file" '$1==f {print $2}' | paste -sd ',' -)
53 - shellcheck -s bash --include="$codes" "$file" 2>&1 || true
54 - fi
55 - done
56 - exit 1
57 -fi
22 +find . -path ./.git -prune -o -name "*.sh" \
23 + -not -path "./.buildkite/scripts/hardware_ci/run-amd-test.sh" -print0 | \
24 + xargs -0 sh -c "for f in \"\$@\"; do git check-ignore -q \"\$f\" || shellcheck -s bash \"\$f\"; done" --
tools/vllm-rocm/generate-rocm-wheels-root-index.sh
+1 -1
@@ -190,7 +190,7 @@ echo ""
190 # List what would be uploaded
191 echo "Files to upload:"
192 find "$WORK_DIR/output" -name "*.html" -type f | while read -r file; do
193 - rel_path="${file#$WORK_DIR/output/}"
193 + rel_path="${file#"$WORK_DIR"/output/}"
194 echo " rocm/$rel_path"
195 done
196 echo ""
tools/vllm-tpu/build.sh
+1 -1
@@ -38,7 +38,7 @@ if ! grep -q "name = \"vllm-tpu\"" "$PYPROJECT_FILE"; then
38 cp "$PYPROJECT_FILE" "${PYPROJECT_FILE}.bak"
39 sed -i '0,/^name = "vllm"/s//name = "vllm-tpu"/' "$PYPROJECT_FILE"
40
41 - echo "Patching ${CHANGE_FILE_LIST[@]} vllm to vllm-tpu..."
41 + echo "Patching ${CHANGE_FILE_LIST[*]} vllm to vllm-tpu..."
42 # patching
43 # importlib.metadata.version('vllm') -> importlib.metadata.version('vllm-tpu')
44 # importlib.metadata.version("vllm") -> importlib.metadata.version("vllm-tpu")