A high-throughput and memory-efficient inference and serving engine for LLMs
[XPU][CI] Refine docker image build and pull/create lock mechanism in Intel GPU CI (#44481)
Signed-off-by: zengxian <xiangdong.zeng@intel.com>
xiangdong committed
Jun 9, 2026 at 20:20 UTC
b12e42d13277a691226e0f6654ad04f790acadd6
2 files changed
+86
-87
.buildkite/scripts/hardware_ci/run-intel-test.sh
+5
-19
@@ -324,23 +324,6 @@ IMAGE="${IMAGE_TAG_XPU:-${image_name}}"
324
325
echo "Using image: ${IMAGE}"
326
327
-if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
328
- echo "Image already exists locally, skipping pull"
329
-else
330
- echo "Image not found locally, waiting for lock..."
331
-
332
- flock /tmp/docker-pull.lock bash -c "
333
- if docker image inspect '${IMAGE}' >/dev/null 2>&1; then
334
- echo 'Image already pulled by another runner'
335
- else
336
- echo 'Pulling image...'
337
- timeout 900 docker pull '${IMAGE}'
338
- fi
339
- "
340
-
341
- echo "Pull step completed"
342
-fi
343
-
327
remove_docker_container() {
328
docker rm -f "${container_name}" || true
329
}
@@ -357,9 +340,12 @@ export HF_TOKEN ZE_AFFINITY_MASK
340
341
{
342
flock 9
360
- if ! docker image inspect "${IMAGE}" >/dev/null 2>&1; then
361
- echo 'Image missing before container creation, pulling again...'
343
+ if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
344
+ echo "Image already exists locally, skipping pull"
345
+ else
346
+ echo "Image not found locally, pulling image..."
347
timeout 900 docker pull "${IMAGE}"
348
+ echo "Pull step completed"
349
fi
350
351
docker create \
docker/Dockerfile.xpu
+81
-68
@@ -89,8 +89,8 @@ RUN mkdir neo && \
89
ENV PATH="/root/.local/bin:$PATH"
90
ENV VIRTUAL_ENV="/opt/venv"
91
ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python
92
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh
93
-RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
92
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh \
93
+ && uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
94
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
95
96
# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.3.
@@ -99,8 +99,8 @@ RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.9/${
99
bash "${ONECCL_INSTALLER}" -a --silent --eula accept && \
100
rm "${ONECCL_INSTALLER}" && \
101
echo "source /opt/intel/oneapi/setvars.sh --force" >> /root/.bashrc && \
102
- echo "source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force" >> /root/.bashrc
103
-RUN rm -f /opt/intel/oneapi/ccl/latest && \
102
+ echo "source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force" >> /root/.bashrc && \
103
+ rm -f /opt/intel/oneapi/ccl/latest && \
104
ln -s /opt/intel/oneapi/ccl/2021.15 /opt/intel/oneapi/ccl/latest
105
106
SHELL ["bash", "-c"]
@@ -119,82 +119,35 @@ ENV UV_LINK_MODE="copy"
119
RUN --mount=type=cache,target=/root/.cache/uv \
120
--mount=type=bind,src=requirements/common.txt,target=/workspace/vllm/requirements/common.txt \
121
--mount=type=bind,src=requirements/xpu.txt,target=/workspace/vllm/requirements/xpu.txt \
122
- --mount=type=bind,src=requirements/test/xpu.txt,target=/workspace/vllm/requirements/test/xpu.txt \
123
- uv pip install --upgrade pip && \
124
- uv pip install -r requirements/xpu.txt && \
125
- uv pip install grpcio-tools protobuf nanobind && \
126
- source /opt/intel/oneapi/setvars.sh --force && \
127
- source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \
128
- export CMAKE_PREFIX_PATH="$(python3 -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}" && \
129
- uv pip install --no-build-isolation -r /workspace/vllm/requirements/test/xpu.txt
122
+ uv pip install --upgrade pip
123
124
125
126
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
134
-
135
-COPY . .
136
-
137
-# Drop the pre-built rust frontend binary into the source tree. setup.py
138
-# detects it and ships it as-is, skipping the local cargo build.
139
-COPY --from=rust-build /workspace/vllm-rs vllm/vllm-rs
140
-
141
-ARG GIT_REPO_CHECK=0
142
-RUN --mount=type=bind,source=.git,target=.git \
143
- if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh; fi
144
-
145
-ENV VLLM_TARGET_DEVICE=xpu
146
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
147
-
148
-RUN --mount=type=cache,target=/root/.cache/uv \
149
- --mount=type=bind,source=.git,target=.git \
150
- uv pip install --no-build-isolation .
151
-
127
CMD ["/bin/bash"]
128
154
-FROM vllm-base AS vllm-openai
155
-
156
-# install development dependencies (for testing)
157
-RUN --mount=type=cache,target=/root/.cache/uv \
158
- uv pip install -e tests/vllm_test_utils
129
+######################### UCX + NIXL BUILD STAGE #########################
130
+# Build UCX and NIXL in a dedicated stage so compiler/autotools layers are
131
+# never included in the final runtime image (mirrors ROCm's build_rixl stage).
132
+FROM vllm-base AS ucx-nixl-build
133
160
-# install NIXL and UCX from source code
134
ARG UCX_VERSION=e5d98879705239d254ede40b4a52891850cb5349
135
ARG NIXL_VERSION=0.7.0
136
164
-RUN apt-get update && apt-get install -y \
165
- pciutils \
166
- net-tools \
167
- iproute2 \
168
- hwloc \
169
- numactl \
170
- wget \
171
- curl \
172
- git \
137
+# Build-time only: compiler, autotools, and verbs dev headers
138
+RUN apt-get update -y && apt-get install -y --no-install-recommends \
139
build-essential \
140
autoconf \
141
automake \
142
libtool \
143
pkg-config \
178
- rdma-core \
144
libibverbs-dev \
180
- ibverbs-utils \
181
- libibverbs1 \
145
librdmacm-dev \
183
- librdmacm1 \
184
- libibumad-dev \
185
- libibumad3 \
186
- libibmad-dev \
187
- libibmad5 \
188
- infiniband-diags \
189
- perftest \
190
- ibutils \
191
- libmlx5-1 \
192
- libmlx4-1 \
193
- ibverbs-providers \
194
- librdmacm1t64
146
+ && rm -rf /var/lib/apt/lists/*
147
196
-ENV PKG_CONFIG_PATH=/tmp/ucx_install/lib/pkgconfig:${PKG_CONFIG_PATH}
197
-ENV LD_LIBRARY_PATH=/tmp/ucx_install/lib:${LD_LIBRARY_PATH}
148
+# Build UCX and produce a NIXL wheel so the final image needs no compiler.
149
+# patchelf (installed via uv) is used by the NIXL wheel build to rewrite
150
+# RPATH entries, making the wheel portable across stages.
151
RUN --mount=type=cache,target=/root/.cache/uv \
152
git clone https://github.com/openucx/ucx /tmp/ucx_source && \
153
cd /tmp/ucx_source && git checkout "${UCX_VERSION}" && \
@@ -203,19 +156,79 @@ RUN --mount=type=cache,target=/root/.cache/uv \
156
make CFLAGS="-Wno-error=incompatible-pointer-types" -j8 && make install && \
157
git clone https://github.com/ai-dynamo/nixl /tmp/nixl_source && \
158
cd /tmp/nixl_source && git checkout "${NIXL_VERSION}" && \
206
- cd /tmp/nixl_source && \
159
uv pip install --upgrade meson pybind11 patchelf && \
160
uv pip install -r requirements.txt && \
209
- uv pip install . && \
161
+ PKG_CONFIG_PATH=/tmp/ucx_install/lib/pkgconfig \
162
+ LD_LIBRARY_PATH=/tmp/ucx_install/lib \
163
+ python -m pip wheel --no-deps . -w /tmp/nixl_wheels/ && \
164
+ find /tmp/ucx_install -type f \( -name '*.a' -o -name '*.la' \) -delete && \
165
+ rm -rf /tmp/ucx_install/include /tmp/ucx_install/share /tmp/ucx_install/etc /tmp/ucx_install/lib/cmake /tmp/ucx_install/bin && \
166
rm -rf /tmp/ucx_source /tmp/nixl_source
167
212
-# FIX triton
168
+FROM vllm-base AS vllm-openai
169
+
170
+ARG UCX_VERSION=e5d98879705239d254ede40b4a52891850cb5349
171
+ARG NIXL_VERSION=0.7.0
172
+
173
+# Copy compiled UCX runtime libraries and the pre-built NIXL wheel.
174
+# No compiler or autotools are installed in this stage.
175
+COPY --from=ucx-nixl-build /tmp/ucx_install /tmp/ucx_install
176
+COPY --from=ucx-nixl-build /tmp/nixl_wheels /tmp/nixl_wheels
177
+
178
+ENV LD_LIBRARY_PATH=/tmp/ucx_install/lib:${LD_LIBRARY_PATH}
179
+
180
+# Install RDMA runtime libraries (no build tools) and the pre-built NIXL wheel.
181
+# Do not uninstall/reinstall large Python packages here to avoid extra layer
182
+# churn; final package resolution remains in the later app install step.
183
+RUN --mount=type=cache,target=/root/.cache/uv \
184
+ apt-get update -y && apt-get install -y --no-install-recommends \
185
+ rdma-core \
186
+ libibverbs1 \
187
+ librdmacm1 \
188
+ libibumad3 \
189
+ libibmad5 \
190
+ libmlx5-1 \
191
+ libmlx4-1 \
192
+ ibverbs-providers \
193
+ librdmacm1t64 \
194
+ && rm -rf /var/lib/apt/lists/* \
195
+ && uv pip install --no-deps /tmp/nixl_wheels/nixl-*.whl \
196
+ && rm -rf /tmp/nixl_wheels
197
+
198
RUN --mount=type=cache,target=/root/.cache/uv \
199
+ --mount=type=bind,src=requirements/common.txt,target=/workspace/vllm/requirements/common.txt \
200
+ --mount=type=bind,src=requirements/xpu.txt,target=/workspace/vllm/requirements/xpu.txt \
201
+ --mount=type=bind,src=requirements/test/xpu.txt,target=/workspace/vllm/requirements/test/xpu.txt \
202
+ uv pip install grpcio-tools protobuf nanobind && \
203
+ uv pip install -r /workspace/vllm/requirements/xpu.txt && \
204
+ uv pip install --no-build-isolation -r /workspace/vllm/requirements/test/xpu.txt && \
205
uv pip uninstall triton triton-xpu && \
215
- uv pip install triton-xpu==3.7.0
206
+ uv pip install triton-xpu==3.7.0 && \
207
+ uv pip uninstall oneccl oneccl-devel && \
208
+ source /opt/intel/oneapi/setvars.sh --force && \
209
+ source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \
210
+ export CMAKE_PREFIX_PATH="$(python3 -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}"
211
+
212
+# Keep source-dependent layers near the end so frequent code-only changes
213
+# don't invalidate heavy dependency and UCX/NIXL layers.
214
+COPY . .
215
+
216
+# Drop the pre-built rust frontend binary into the source tree. setup.py
217
+# detects it and ships it as-is, skipping the local cargo build.
218
+COPY --from=rust-build /workspace/vllm-rs vllm/vllm-rs
219
+
220
+ARG GIT_REPO_CHECK=0
221
+RUN --mount=type=bind,source=.git,target=.git \
222
+ if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh; fi
223
+
224
+ENV VLLM_TARGET_DEVICE=xpu
225
+ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
226
217
-# remove torch bundled oneccl to avoid conflicts
227
RUN --mount=type=cache,target=/root/.cache/uv \
219
- uv pip uninstall oneccl oneccl-devel
228
+ --mount=type=bind,source=.git,target=.git \
229
+ uv pip install --no-build-isolation --no-deps .
230
+
231
+RUN --mount=type=cache,target=/root/.cache/uv \
232
+ uv pip install -e tests/vllm_test_utils
233
234
ENTRYPOINT ["vllm", "serve"]