@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
sh 162 lines 6.03 KB
Raw
1 #!/bin/bash
2
3 # This script build the Ascend NPU docker image and run the offline inference inside the container.
4 # It serves a sanity check for compilation and basic model usage.
5 set -ex
6
7 # Base ubuntu image with basic ascend development libraries and python installed
8 VLLM_ASCEND_REPO="https://github.com/vllm-project/vllm-ascend.git"
9 CONFIG_FILE_REMOTE_PATH="tests/e2e/vllm_interface/vllm_test.cfg"
10 TEST_RUN_CONFIG_FILE="vllm_test.cfg"
11 VLLM_ASCEND_TMP_DIR=
12 # Get the test run configuration file from the vllm-ascend repository
13 fetch_vllm_test_cfg() {
14 VLLM_ASCEND_TMP_DIR=$(mktemp -d)
15 # Ensure that the temporary directory is cleaned up when an exception occurs during configuration file retrieval
16 cleanup() {
17 rm -rf "${VLLM_ASCEND_TMP_DIR}"
18 }
19 trap cleanup EXIT
20
21 GIT_TRACE=1 git clone -v --depth 1 "${VLLM_ASCEND_REPO}" "${VLLM_ASCEND_TMP_DIR}"
22 if [ ! -f "${VLLM_ASCEND_TMP_DIR}/${CONFIG_FILE_REMOTE_PATH}" ]; then
23 echo "Error: file '${CONFIG_FILE_REMOTE_PATH}' does not exist in the warehouse" >&2
24 exit 1
25 fi
26
27 # If the file already exists locally, just overwrite it
28 cp "${VLLM_ASCEND_TMP_DIR}/${CONFIG_FILE_REMOTE_PATH}" "${TEST_RUN_CONFIG_FILE}"
29 echo "Copied ${CONFIG_FILE_REMOTE_PATH} to ${TEST_RUN_CONFIG_FILE}"
30
31 # Since the trap will be overwritten later, and when it is executed here, the task of cleaning up resources
32 # when the trap is abnormal has been completed, so the temporary resources are manually deleted here.
33 rm -rf "${VLLM_ASCEND_TMP_DIR}"
34 trap - EXIT
35 }
36
37 # Downloads test run configuration file from a remote URL.
38 # Loads the configuration into the current script environment.
39 get_config() {
40 if [ ! -f "${TEST_RUN_CONFIG_FILE}" ]; then
41 echo "Error: file '${TEST_RUN_CONFIG_FILE}' does not exist in the warehouse" >&2
42 exit 1
43 fi
44 # shellcheck source=/dev/null
45 source "${TEST_RUN_CONFIG_FILE}"
46 echo "Base docker image name that get from configuration: ${BASE_IMAGE_NAME}"
47 return 0
48 }
49
50 # get test running configuration.
51 fetch_vllm_test_cfg
52 # Check if the function call was successful. If not, exit the script.
53 if ! get_config; then
54 exit 1
55 fi
56
57 image_name="npu/vllm-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}"
58 container_name="npu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
59
60 # BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards
61 agent_idx=$(echo "${BUILDKITE_AGENT_NAME}" | awk -F'-' '{print $(NF-1)}')
62 echo "agent_idx: ${agent_idx}"
63 builder_name="cachebuilder${agent_idx}"
64 builder_cache_dir="/mnt/docker-cache${agent_idx}"
65 mkdir -p "${builder_cache_dir}"
66
67 # Try building the docker image
68 cat <<EOF | DOCKER_BUILDKIT=1 docker build \
69 --add-host cache-service-vllm.nginx-pypi-cache.svc.cluster.local:"${PYPI_CACHE_HOST}" \
70 --builder "${builder_name}" --cache-from type=local,src="${builder_cache_dir}" \
71 --cache-to type=local,dest="${builder_cache_dir}",mode=max \
72 --progress=plain --load -t "${image_name}" -f - .
73 FROM ${BASE_IMAGE_NAME}
74
75 # Define environments
76 ENV DEBIAN_FRONTEND=noninteractive
77 ENV SOC_VERSION="ascend910b1"
78
79 RUN pip config set global.index-url http://cache-service-vllm.nginx-pypi-cache.svc.cluster.local:${PYPI_CACHE_PORT}/pypi/simple && \
80 pip config set global.trusted-host cache-service-vllm.nginx-pypi-cache.svc.cluster.local
81
82 # Install for pytest to make the docker build cache layer always valid
83 RUN --mount=type=cache,target=/root/.cache/pip \
84 pip install pytest>=6.0
85
86 WORKDIR /workspace/vllm
87
88 COPY . .
89
90 # Install vllm-ascend
91 WORKDIR /workspace
92 ARG VLLM_ASCEND_REPO=https://github.com/vllm-project/vllm-ascend.git
93 ARG VLLM_ASCEND_TAG=main
94 RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/" && \
95 git clone --depth 1 \$VLLM_ASCEND_REPO --branch \$VLLM_ASCEND_TAG /workspace/vllm-ascend
96
97 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
98 ENV VLLM_USE_MODELSCOPE=True
99
100 WORKDIR /workspace/vllm-ascend
101
102 CMD ["/bin/bash"]
103
104 EOF
105
106 # Setup cleanup
107 remove_docker_container() {
108 docker rm -f "${container_name}" || true;
109 docker image rm -f "${image_name}" || true;
110 docker system prune -f || true;
111 }
112 trap remove_docker_container EXIT
113
114 # Generate corresponding --device args based on BUILDKITE_AGENT_NAME
115 # Ascend NPU BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards, and agent_idx starts from 1.
116 # e.g. atlas-a2-001-1-2cards means this is the 1-th agent on atlas-a2-001 host, and it has 2 NPU cards.
117 # returns one argument per line: --device, /dev/davinciX, ...
118 parse_and_gen_devices() {
119 local input="$1"
120 local index cards_num
121 if [[ "$input" =~ ([0-9]+)-([0-9]+)cards$ ]]; then
122 index="${BASH_REMATCH[1]}"
123 cards_num="${BASH_REMATCH[2]}"
124 else
125 echo "parse error" >&2
126 return 1
127 fi
128
129 local i=0
130 while (( i < cards_num )); do
131 local dev_idx=$(((index - 1)*cards_num + i ))
132 printf '%s\n' "--device"
133 printf '%s\n' "/dev/davinci${dev_idx}"
134 ((i++))
135 done
136 }
137
138 mapfile -t device_args < <(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1
139
140 # Run the image and execute the Out-Of-Tree (OOT) platform interface test case on Ascend NPU hardware.
141 # This test checks whether the OOT platform interface is functioning properly in conjunction with
142 # the hardware plugin vllm-ascend.
143 model_cache_dir=/mnt/modelscope${agent_idx}
144 mkdir -p "${model_cache_dir}"
145 docker run \
146 "${device_args[@]}" \
147 --device /dev/davinci_manager \
148 --device /dev/devmm_svm \
149 --device /dev/hisi_hdc \
150 -v /usr/local/dcmi:/usr/local/dcmi \
151 -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
152 -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
153 -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
154 -v /etc/ascend_install.info:/etc/ascend_install.info \
155 -v "${model_cache_dir}":/root/.cache/modelscope \
156 --entrypoint="" \
157 --name "${container_name}" \
158 "${image_name}" \
159 bash -c '
160 set -e
161 pytest --noconftest -v -s tests/e2e/vllm_interface/test_vllm_pr_interface_compatibility.py
162 '