A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 13:00 UTC
[Docker][KVConnector] Build mooncake-transfer-engine from source (#42114)
Signed-off-by: Zhewen Li <zhewenli@inferact.ai> Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Zhewen Li <zhewenli@inferact.ai> Co-authored-by: khluu <khluu000@gmail.com>
Zhewen Li committed
May 16, 2026 at 00:26 UTC
657b42b5922d21fef00529144ef5bb5633ad04b1
2 files changed
+41
.buildkite/release-pipeline.yaml
+19
@@ -7,6 +7,9 @@ env:
7
CUDA_ARCH_AARCH64: "8.0 8.7 8.9 9.0 10.0 11.0 12.0+PTX"
8
CUDA_ARCH_X86_CU129: "7.5 8.0 8.6 8.9 9.0 10.0 12.0"
9
CUDA_ARCH_AARCH64_CU129: "8.0 8.7 8.9 9.0 10.0 12.0"
10
+ MOONCAKE_WHEEL_AARCH64_2_35: "https://vllm-wheels.s3.amazonaws.com/mooncake/mooncake_transfer_engine-0.3.10.post2-0da9dfea3-cp312-cp312-manylinux_2_35_aarch64.whl"
11
+ MOONCAKE_WHEEL_AARCH64_2_39: "https://vllm-wheels.s3.amazonaws.com/mooncake/mooncake_transfer_engine-0.3.10.post2-0da9dfea3-cp312-cp312-manylinux_2_39_aarch64.whl"
12
+ MOONCAKE_WHEEL_X86_64: "https://vllm-wheels.s3.amazonaws.com/mooncake/mooncake_transfer_engine-0.3.10.post2-0da9dfea3-cp312-cp312-manylinux_2_35_x86_64.whl"
13
14
steps:
15
- input: "Provide Release version here"
@@ -136,6 +139,8 @@ steps:
139
--build-arg CUDA_VERSION=13.0.2 \
140
--build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \
141
--build-arg INSTALL_KV_CONNECTORS=true \
142
+ --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \
143
+ --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \
144
--build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 \
145
--target vllm-openai \
146
--progress plain \
@@ -162,6 +167,8 @@ steps:
167
--build-arg CUDA_VERSION=13.0.2 \
168
--build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \
169
--build-arg INSTALL_KV_CONNECTORS=true \
170
+ --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \
171
+ --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \
172
--build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 \
173
--target vllm-openai \
174
--progress plain \
@@ -185,6 +192,8 @@ steps:
192
--build-arg CUDA_VERSION=12.9.1 \
193
--build-arg torch_cuda_arch_list="${CUDA_ARCH_X86_CU129}" \
194
--build-arg INSTALL_KV_CONNECTORS=true \
195
+ --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \
196
+ --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \
197
--target vllm-openai \
198
--progress plain \
199
-f docker/Dockerfile .
@@ -210,6 +219,8 @@ steps:
219
--build-arg CUDA_VERSION=12.9.1 \
220
--build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64_CU129}" \
221
--build-arg INSTALL_KV_CONNECTORS=true \
222
+ --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \
223
+ --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \
224
--target vllm-openai \
225
--progress plain \
226
-f docker/Dockerfile .
@@ -234,6 +245,8 @@ steps:
245
--build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \
246
--build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \
247
--build-arg INSTALL_KV_CONNECTORS=true \
248
+ --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \
249
+ --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \
250
--build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 \
251
--target vllm-openai \
252
--progress plain \
@@ -261,6 +274,8 @@ steps:
274
--build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \
275
--build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \
276
--build-arg INSTALL_KV_CONNECTORS=true \
277
+ --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \
278
+ --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \
279
--build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 \
280
--target vllm-openai \
281
--progress plain \
@@ -286,6 +301,8 @@ steps:
301
--build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \
302
--build-arg torch_cuda_arch_list="${CUDA_ARCH_X86_CU129}" \
303
--build-arg INSTALL_KV_CONNECTORS=true \
304
+ --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \
305
+ --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \
306
--target vllm-openai \
307
--progress plain \
308
-f docker/Dockerfile .
@@ -312,6 +329,8 @@ steps:
329
--build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \
330
--build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64_CU129}" \
331
--build-arg INSTALL_KV_CONNECTORS=true \
332
+ --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \
333
+ --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \
334
--target vllm-openai \
335
--progress plain \
336
-f docker/Dockerfile .
docker/Dockerfile
+22
@@ -858,6 +858,28 @@ RUN --mount=type=cache,target=/root/.cache/uv \
858
uv pip install --system --force-reinstall --no-deps nixl-cu${CUDA_MAJOR}; \
859
fi
860
861
+# Optional override: install mooncake-transfer-engine from a URL instead of the
862
+# PyPI release pulled in above. Use this for wheels built with non-default CMake
863
+# flags (e.g. `STORE_USE_ETCD=ON` for master HA). The URL's manylinux glibc
864
+# floor must be <= the FINAL_BASE_IMAGE's glibc.
865
+ARG MOONCAKE_WHEEL_AARCH64
866
+ARG MOONCAKE_WHEEL_X86_64
867
+RUN if [ "$INSTALL_KV_CONNECTORS" = "true" ]; then \
868
+ if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
869
+ WHEEL="${MOONCAKE_WHEEL_AARCH64}"; \
870
+ else \
871
+ WHEEL="${MOONCAKE_WHEEL_X86_64}"; \
872
+ fi && \
873
+ if [ -n "${WHEEL}" ]; then \
874
+ uv pip install --system "${WHEEL}" && \
875
+ CUDA_MAJOR="${CUDA_VERSION%%.*}" && \
876
+ if [ ! -f /usr/local/cuda/lib64/libcudart.so ] && \
877
+ [ -f "/usr/local/cuda/lib64/libcudart.so.${CUDA_MAJOR}" ]; then \
878
+ ln -s "libcudart.so.${CUDA_MAJOR}" /usr/local/cuda/lib64/libcudart.so; \
879
+ fi; \
880
+ fi; \
881
+ fi
882
+
883
ENV VLLM_USAGE_SOURCE production-docker-image
884
ENV VLLM_BUILD_COMMIT=${VLLM_BUILD_COMMIT:-unknown} \
885
VLLM_BUILD_PIPELINE=${VLLM_BUILD_PIPELINE:-local} \