A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[CI] add kvcache-connector dependency definition and add into CI build (#18193)
Signed-off-by: Peter Pan <Peter.Pan@daocloud.io>
Peter Pan committed
Jul 4, 2025 at 21:49 UTC
5561681d0400469aba6abfee242be717db47ae0c
3 files changed
+14
-1
.buildkite/release-pipeline.yaml
+1
-1
@@ -52,7 +52,7 @@ steps:
52
queue: cpu_queue_postmerge
53
commands:
54
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
55
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.8.1 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT --target vllm-openai --progress plain -f docker/Dockerfile ."
55
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.8.1 --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT --target vllm-openai --progress plain -f docker/Dockerfile ."
56
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
57
58
- label: "Annotate release workflow"
docker/Dockerfile
+12
@@ -1,3 +1,4 @@
1
+
2
# The vLLM Dockerfile is used to construct vLLM image that can be directly used
3
# to run the OpenAI compatible server.
4
@@ -62,12 +63,16 @@ ARG PYTORCH_CUDA_NIGHTLY_INDEX_BASE_URL=https://download.pytorch.org/whl/nightly
63
ARG PIP_KEYRING_PROVIDER=disabled
64
ARG UV_KEYRING_PROVIDER=${PIP_KEYRING_PROVIDER}
65
66
+# Flag enables build-in KV-connector dependency libs into docker images
67
+ARG INSTALL_KV_CONNECTORS=false
68
+
69
#################### BASE BUILD IMAGE ####################
70
# prepare basic build environment
71
FROM ${BUILD_BASE_IMAGE} AS base
72
ARG CUDA_VERSION
73
ARG PYTHON_VERSION
74
ARG TARGETPLATFORM
75
+ARG INSTALL_KV_CONNECTORS=false
76
ENV DEBIAN_FRONTEND=noninteractive
77
78
ARG DEADSNAKES_MIRROR_URL
@@ -276,6 +281,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
281
FROM ${FINAL_BASE_IMAGE} AS vllm-base
282
ARG CUDA_VERSION
283
ARG PYTHON_VERSION
284
+ARG INSTALL_KV_CONNECTORS=false
285
WORKDIR /vllm-workspace
286
ENV DEBIAN_FRONTEND=noninteractive
287
ARG TARGETPLATFORM
@@ -485,6 +491,7 @@ RUN mv mkdocs.yaml test_docs/
491
# base openai image with additional requirements, for any subsequent openai-style images
492
FROM vllm-base AS vllm-openai-base
493
ARG TARGETPLATFORM
494
+ARG INSTALL_KV_CONNECTORS=false
495
496
ARG PIP_INDEX_URL UV_INDEX_URL
497
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
@@ -493,8 +500,13 @@ ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
500
# Reference: https://github.com/astral-sh/uv/pull/1694
501
ENV UV_HTTP_TIMEOUT=500
502
503
+COPY requirements/kv_connectors.txt requirements/kv_connectors.txt
504
+
505
# install additional dependencies for openai api server
506
RUN --mount=type=cache,target=/root/.cache/uv \
507
+ if [ "$INSTALL_KV_CONNECTORS" = "true" ]; then \
508
+ uv pip install --system -r requirements/kv_connectors.txt; \
509
+ fi; \
510
if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
511
uv pip install --system accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.42.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
512
else \
requirements/kv_connectors.txt
new
+1
@@ -0,0 +1 @@
1
+lmcache
\ No newline at end of file