A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
Simplify ep kernels installation (#19412)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jun 10, 2025 at 20:06 UTC
64a9af5afa976a4cfdbbd5b63ff0ea88953f4203
5 files changed
+26
-69
tools/ep_kernels/README.md
+4
-6
@@ -1,11 +1,10 @@
1
Large-scale cluster-level expert parallel, as described in the [DeepSeek-V3 Technical Report](http://arxiv.org/abs/2412.19437), is an efficient way to deploy sparse MoE models with many experts. However, such deployment requires many components beyond a normal Python package, including system package support and system driver support. It is impossible to bundle all these components into a Python package.
2
3
-Here we break down the requirements in 3 steps:
3
+Here we break down the requirements in 2 steps:
4
1. Build and install the Python libraries (both [pplx-kernels](https://github.com/ppl-ai/pplx-kernels) and [DeepEP](https://github.com/deepseek-ai/DeepEP)), including necessary dependencies like NVSHMEM. This step does not require any privileged access. Any user can do this.
5
-2. Build and install the system libraries (GDR Copy). This step requires root access. You can do it inside a Docker container so that they can be shipped as a single image.
6
-3. Build and install the system drivers (GDR Copy, and necessary modifications to NVIDIA driver to enable IBGDA). This step requires root access, and must be done on the host machine.
5
+2. Configure NVIDIA driver to enable IBGDA. This step requires root access, and must be done on the host machine.
6
8
-2 and 3 are necessary for multi-node deployment.
7
+2 is necessary for multi-node deployment.
8
9
All scripts accept a positional argument as workspace path for staging the build, defaulting to `$(pwd)/ep_kernels_workspace`.
10
@@ -21,7 +20,6 @@ bash install_python_libraries.sh
20
21
```bash
22
bash install_python_libraries.sh
24
-sudo bash install_system_libraries.sh
25
-sudo bash install_system_drivers.sh
23
+sudo bash configure_system_drivers.sh
24
sudo reboot # Reboot is required to load the new driver
25
```
tools/ep_kernels/configure_system_drivers.sh
new
+7
@@ -0,0 +1,7 @@
1
+set -ex
2
+
3
+# turn on IBGDA
4
+echo 'options nvidia NVreg_EnableStreamMemOPs=1 NVreg_RegistryDwords="PeerMappingOverride=1;"' | tee -a /etc/modprobe.d/nvidia.conf
5
+update-initramfs -u
6
+
7
+echo "Please reboot the system to apply the changes"
tools/ep_kernels/install_python_libraries.sh
+15
-21
@@ -13,16 +13,6 @@ fi
13
# install dependencies if not installed
14
pip3 install cmake torch ninja
15
16
-# build gdrcopy, required by nvshmem
17
-pushd $WORKSPACE
18
-wget https://github.com/NVIDIA/gdrcopy/archive/refs/tags/v2.4.4.tar.gz
19
-mkdir -p gdrcopy_src
20
-tar -xvf v2.4.4.tar.gz -C gdrcopy_src --strip-components=1
21
-pushd gdrcopy_src
22
-make -j$(nproc)
23
-make prefix=$WORKSPACE/gdrcopy_install install
24
-popd
25
-
16
# build nvshmem
17
pushd $WORKSPACE
18
mkdir -p nvshmem_src
@@ -34,26 +24,30 @@ git init
24
git apply -vvv nvshmem.patch
25
26
# assume CUDA_HOME is set correctly
37
-export GDRCOPY_HOME=$WORKSPACE/gdrcopy_install
27
+if [ -z "$CUDA_HOME" ]; then
28
+ echo "CUDA_HOME is not set, please set it to your CUDA installation directory."
29
+ exit 1
30
+fi
31
+
32
+# disable all features except IBGDA
33
+export NVSHMEM_IBGDA_SUPPORT=1
34
+
35
export NVSHMEM_SHMEM_SUPPORT=0
36
export NVSHMEM_UCX_SUPPORT=0
37
export NVSHMEM_USE_NCCL=0
41
-export NVSHMEM_IBGDA_SUPPORT=1
38
export NVSHMEM_PMIX_SUPPORT=0
39
export NVSHMEM_TIMEOUT_DEVICE_POLLING=0
44
-export NVSHMEM_USE_GDRCOPY=1
45
-export NVSHMEM_IBRC_SUPPORT=1
46
-
47
-# remove MPI dependency
40
+export NVSHMEM_USE_GDRCOPY=0
41
+export NVSHMEM_IBRC_SUPPORT=0
42
export NVSHMEM_BUILD_TESTS=0
43
export NVSHMEM_BUILD_EXAMPLES=0
44
export NVSHMEM_MPI_SUPPORT=0
45
+export NVSHMEM_BUILD_HYDRA_LAUNCHER=0
46
+export NVSHMEM_BUILD_TXZ_PACKAGE=0
47
+export NVSHMEM_TIMEOUT_DEVICE_POLLING=0
48
52
-cmake -S . -B $WORKSPACE/nvshmem_build/ -DCMAKE_INSTALL_PREFIX=$WORKSPACE/nvshmem_install
53
-
54
-cd $WORKSPACE/nvshmem_build/
55
-make -j$(nproc)
56
-make install
49
+cmake -G Ninja -S . -B $WORKSPACE/nvshmem_build/ -DCMAKE_INSTALL_PREFIX=$WORKSPACE/nvshmem_install
50
+cmake --build $WORKSPACE/nvshmem_build/ --target install
51
52
popd
53
tools/ep_kernels/install_system_drivers.sh
deleted
-24
@@ -1,24 +0,0 @@
1
-set -ex
2
-
3
-# prepare workspace directory
4
-WORKSPACE=$1
5
-if [ -z "$WORKSPACE" ]; then
6
- export WORKSPACE=$(pwd)/ep_kernels_workspace
7
-fi
8
-
9
-if [ ! -d "$WORKSPACE" ]; then
10
- mkdir -p $WORKSPACE
11
-fi
12
-
13
-# build and install gdrcopy driver
14
-pushd $WORKSPACE
15
-cd gdrcopy_src
16
-./insmod.sh
17
-# run gdrcopy_copybw to test the installation
18
-$WORKSPACE/gdrcopy_install/bin/gdrcopy_copybw
19
-
20
-# turn on IBGDA
21
-echo 'options nvidia NVreg_EnableStreamMemOPs=1 NVreg_RegistryDwords="PeerMappingOverride=1;"' | tee -a /etc/modprobe.d/nvidia.conf
22
-update-initramfs -u
23
-
24
-echo "Please reboot the system to apply the changes"
tools/ep_kernels/install_system_libraries.sh
deleted
-18
@@ -1,18 +0,0 @@
1
-set -ex
2
-
3
-# prepare workspace directory
4
-WORKSPACE=$1
5
-if [ -z "$WORKSPACE" ]; then
6
- export WORKSPACE=$(pwd)/ep_kernels_workspace
7
-fi
8
-
9
-if [ ! -d "$WORKSPACE" ]; then
10
- mkdir -p $WORKSPACE
11
-fi
12
-
13
-# build and install gdrcopy system packages
14
-pushd $WORKSPACE
15
-cd gdrcopy_src/packages
16
-apt install devscripts -y
17
-CUDA=${CUDA_HOME:-/usr/local/cuda} ./build-deb-packages.sh
18
-dpkg -i *.deb