A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Removes source compilation of nixl dependency (#24874)
Signed-off-by: bbartels <benjamin@bartels.dev> Signed-off-by: Benjamin Bartels <benjamin@bartels.dev> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Daniele <36171005+dtrifiro@users.noreply.github.com>
Benjamin Bartels committed
Sep 17, 2025 at 02:33 UTC
64ad551878e9f0323407928fa747ce14c67c71f2
5 files changed
+77
-116
docker/Dockerfile
+16
-4
@@ -283,6 +283,10 @@ WORKDIR /vllm-workspace
283
ENV DEBIAN_FRONTEND=noninteractive
284
ARG TARGETPLATFORM
285
286
+ARG GDRCOPY_CUDA_VERSION=12.8
287
+# Keep in line with FINAL_BASE_IMAGE
288
+ARG GDRCOPY_OS_VERSION=Ubuntu22_04
289
+
290
SHELL ["/bin/bash", "-c"]
291
292
ARG DEADSNAKES_MIRROR_URL
@@ -441,13 +445,21 @@ COPY tools/install_deepgemm.sh /tmp/install_deepgemm.sh
445
RUN --mount=type=cache,target=/root/.cache/uv \
446
VLLM_DOCKER_BUILD_CONTEXT=1 /tmp/install_deepgemm.sh --cuda-version "${CUDA_VERSION}" ${DEEPGEMM_GIT_REF:+--ref "$DEEPGEMM_GIT_REF"}
447
444
-# Install EP kernels(pplx-kernels and DeepEP), NixL
448
+COPY tools/install_gdrcopy.sh install_gdrcopy.sh
449
+RUN set -eux; \
450
+ case "${TARGETPLATFORM}" in \
451
+ linux/arm64) UUARCH="aarch64" ;; \
452
+ linux/amd64) UUARCH="x64" ;; \
453
+ *) echo "Unsupported TARGETPLATFORM: ${TARGETPLATFORM}" >&2; exit 1 ;; \
454
+ esac; \
455
+ ./install_gdrcopy.sh "${GDRCOPY_OS_VERSION}" "${GDRCOPY_CUDA_VERSION}" "${UUARCH}"; \
456
+ rm ./install_gdrcopy.sh
457
+
458
+# Install EP kernels(pplx-kernels and DeepEP)
459
COPY tools/ep_kernels/install_python_libraries.sh install_python_libraries.sh
446
-COPY tools/install_nixl.sh install_nixl.sh
460
ENV CUDA_HOME=/usr/local/cuda
461
RUN export TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-9.0a+PTX}" \
449
- && bash install_python_libraries.sh \
450
- && bash install_nixl.sh --force
462
+ && bash install_python_libraries.sh
463
464
#################### vLLM installation IMAGE ####################
465
docs/serving/expert_parallel_deployment.md
+2
-2
@@ -10,7 +10,7 @@ Before using EP, you need to install the necessary dependencies. We are actively
10
11
1. **Install DeepEP and pplx-kernels**: Set up host environment following vLLM's guide for EP kernels [here](gh-file:tools/ep_kernels).
12
2. **Install DeepGEMM library**: Follow the [official instructions](https://github.com/deepseek-ai/DeepGEMM#installation).
13
-3. **For disaggregated serving**: Install UCX and NIXL following the [script](gh-file:tools/install_nixl.sh).
13
+3. **For disaggregated serving**: Install `gdrcopy` by running the [`install_gdrcopy.sh`](gh-file:tools/install_gdrcopy.sh) script (e.g., `install_gdrcopy.sh "${GDRCOPY_OS_VERSION}" "12.8" "x64"`). You can find available OS versions [here](https://developer.download.nvidia.com/compute/redist/gdrcopy/CUDA%2012.8/).
14
15
### Backend Selection Guide
16
@@ -191,7 +191,7 @@ For production deployments requiring strict SLA guarantees for time-to-first-tok
191
192
### Setup Steps
193
194
-1. **Install KV Connector**: Install NIXL using the [installation script](gh-file:tools/install_nixl.sh)
194
+1. **Install gdrcopy/ucx/nixl**: For maximum performance, run the [install_gdrcopy.sh](gh-file:tools/install_gdrcopy.sh) script to install `gdrcopy` (e.g., `install_gdrcopy.sh "${GDRCOPY_OS_VERSION}" "12.8" "x64"`). You can find available OS versions [here](https://developer.download.nvidia.com/compute/redist/gdrcopy/CUDA%2012.8/). If `gdrcopy` is not installed, things will still work with a plain `pip install nixl`, just with lower performance. `nixl` and `ucx` are installed as dependencies via pip.
195
196
2. **Configure Both Instances**: Add this flag to both prefill and decode instances `--kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}`
197
requirements/kv_connectors.txt
+2
-1
@@ -1 +1,2 @@
1
-lmcache
\ No newline at end of file
1
+lmcache
2
+nixl >= 0.5.1 # Required for disaggregated prefill
tools/install_gdrcopy.sh
new
+57
@@ -0,0 +1,57 @@
1
+#!/usr/bin/env bash
2
+set -euo pipefail
3
+
4
+# Usage: install_gdrcopy.sh <GDRCOPY_OS_VERSION> <GDRCOPY_CUDA_VERSION> <uuarch>
5
+# uuarch must be "x64" or "aarch64"
6
+# Optional: set GDRCOPY_VERSION to override the libgdrapi package version (default: 2.5.1-1)
7
+# Requires: curl, apt-get, root privileges
8
+if [[ $(id -u) -ne 0 ]]; then
9
+ echo "Must be run as root" >&2
10
+
11
+ exit 1
12
+fi
13
+if [[ $# -ne 3 ]]; then
14
+ echo "Usage: $0 <GDRCOPY_OS_VERSION> <GDRCOPY_CUDA_VERSION> <uuarch(x64|aarch64)>" >&2
15
+ exit 1
16
+fi
17
+
18
+OS_VER="$1"
19
+CUDA_VER="$2"
20
+UUARCH_RAW="$3"
21
+
22
+# Normalize/validate arch
23
+case "${UUARCH_RAW,,}" in
24
+ aarch64|arm64)
25
+ URL_ARCH="aarch64"
26
+ DEB_ARCH="arm64"
27
+ ;;
28
+ x64|x86_64|amd64)
29
+ URL_ARCH="x64"
30
+ DEB_ARCH="amd64"
31
+ ;;
32
+ *)
33
+ echo "Unsupported uuarch: ${UUARCH_RAW}. Use 'x64' or 'aarch64'." >&2
34
+ exit 1
35
+ ;;
36
+esac
37
+
38
+OS_VER_LOWER="$(tr '[:upper:]' '[:lower:]' <<<"$OS_VER")"
39
+GDRCOPY_PKG_VER="${GDRCOPY_VERSION:-2.5.1-1}"
40
+
41
+DEB_NAME="libgdrapi_${GDRCOPY_PKG_VER}_${DEB_ARCH}.${OS_VER}.deb"
42
+BASE_URL="https://developer.download.nvidia.com/compute/redist/gdrcopy"
43
+URL="${BASE_URL}/CUDA%20${CUDA_VER}/${OS_VER_LOWER}/${URL_ARCH}/${DEB_NAME}"
44
+
45
+echo "Downloading: ${URL}"
46
+TMPDIR="$(mktemp -d)"
47
+trap 'rm -rf "${TMPDIR}"' EXIT
48
+
49
+curl -fSL "${URL}" -o "${TMPDIR}/${DEB_NAME}"
50
+
51
+export DEBIAN_FRONTEND=noninteractive
52
+apt-get update
53
+apt-get install -y "${TMPDIR}/${DEB_NAME}"
54
+apt-get clean
55
+rm -rf /var/lib/apt/lists/*
56
+
57
+echo "Installed ${DEB_NAME}"
tools/install_nixl.sh
deleted
-109
@@ -1,109 +0,0 @@
1
-#!/bin/bash
2
-# Usage: ./install_nixl.sh [--force]
3
-
4
-FORCE=false
5
-if [ "$1" == "--force" ]; then
6
- FORCE=true
7
-fi
8
-
9
-SUDO=false
10
-if command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then
11
- SUDO=true
12
-fi
13
-
14
-ARCH=$(uname -m)
15
-
16
-ROOT_DIR="/usr/local"
17
-mkdir -p "$ROOT_DIR"
18
-GDR_HOME="$ROOT_DIR/gdrcopy"
19
-UCX_HOME="$ROOT_DIR/ucx"
20
-NIXL_HOME="$ROOT_DIR/nixl"
21
-CUDA_HOME=/usr/local/cuda
22
-
23
-export PATH="$GDR_HOME/bin:$UCX_HOME/bin:$NIXL_HOME/bin:$PATH"
24
-export LD_LIBRARY_PATH="$GDR_HOME/lib:$UCX_HOME/lib:$NIXL_HOME/lib/$ARCH-linux-gnu:$LD_LIBRARY_PATH"
25
-
26
-TEMP_DIR="nixl_installer"
27
-mkdir -p "$TEMP_DIR"
28
-cd "$TEMP_DIR"
29
-
30
-pip install meson ninja pybind11
31
-
32
-if [ ! -e "/dev/gdrdrv" ] || [ "$FORCE" = true ]; then
33
- echo "Installing gdrcopy\n"
34
- wget https://github.com/NVIDIA/gdrcopy/archive/refs/tags/v2.5.tar.gz
35
- tar xzf v2.5.tar.gz; rm v2.5.tar.gz
36
- cd gdrcopy-2.5
37
- make prefix=$GDR_HOME CUDA=$CUDA_HOME all install
38
-
39
- if $SUDO; then
40
- echo "Running insmod.sh with sudo"
41
- sudo ./insmod.sh
42
- else
43
- echo "Skipping insmod.sh - sudo not available"
44
- echo "Please run 'sudo ./gdrcopy-2.5/insmod.sh' manually if needed"
45
- fi
46
-
47
- cd ..
48
-else
49
- echo "Found /dev/gdrdrv. Skipping gdrcopy installation"
50
-fi
51
-
52
-if ! command -v ucx_info &> /dev/null || [ "$FORCE" = true ]; then
53
- echo "Installing UCX"
54
- wget https://github.com/openucx/ucx/releases/download/v1.18.0/ucx-1.18.0.tar.gz
55
- tar xzf ucx-1.18.0.tar.gz; rm ucx-1.18.0.tar.gz
56
- cd ucx-1.18.0
57
-
58
- # Checking Mellanox NICs
59
- MLX_OPTS=""
60
- if lspci | grep -i mellanox > /dev/null || command -v ibstat > /dev/null; then
61
- echo "Mellanox NIC detected, adding Mellanox-specific options"
62
- MLX_OPTS="--with-rdmacm \
63
- --with-mlx5-dv \
64
- --with-ib-hw-tm"
65
- fi
66
-
67
- ./configure --prefix=$UCX_HOME \
68
- --enable-shared \
69
- --disable-static \
70
- --disable-doxygen-doc \
71
- --enable-optimizations \
72
- --enable-cma \
73
- --enable-devel-headers \
74
- --with-cuda=$CUDA_HOME \
75
- --with-dm \
76
- --with-gdrcopy=$GDR_HOME \
77
- --with-verbs \
78
- --enable-mt \
79
- $MLX_OPTS
80
- make -j
81
- make -j install-strip
82
-
83
- if $SUDO; then
84
- echo "Running ldconfig with sudo"
85
- sudo ldconfig
86
- else
87
- echo "Skipping ldconfig - sudo not available"
88
- echo "Please run 'sudo ldconfig' manually if needed"
89
- fi
90
-
91
- cd ..
92
-else
93
- echo "Found existing UCX. Skipping UCX installation"
94
-fi
95
-
96
-if ! command -v nixl_test &> /dev/null || [ "$FORCE" = true ]; then
97
- echo "Installing NIXL"
98
- wget https://github.com/ai-dynamo/nixl/archive/refs/tags/0.2.0.tar.gz
99
- tar xzf 0.2.0.tar.gz; rm 0.2.0.tar.gz
100
- cd nixl-0.2.0
101
- meson setup build --prefix=$NIXL_HOME -Ducx_path=$UCX_HOME
102
- cd build
103
- ninja
104
- ninja install
105
-
106
- cd ../..
107
-else
108
- echo "Found existing NIXL. Skipping NIXL installation"
109
-fi