@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ca81ff5196
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] manage nccl via a pypi package & upgrade to pt 2.2.1 (#3805)

youkaichao committed Apr 4, 2024 at 10:26 UTC ca81ff5196b2fa82b7a9d553cd6e30eab9e72aca
8 files changed +36 -11
.github/workflows/publish.yml
+1 -1
@@ -49,7 +49,7 @@ jobs:
49 matrix:
50 os: ['ubuntu-20.04']
51 python-version: ['3.8', '3.9', '3.10', '3.11']
52 - pytorch-version: ['2.1.2'] # Must be the most recent version that meets requirements.txt.
52 + pytorch-version: ['2.2.1'] # Must be the most recent version that meets requirements.txt.
53 cuda-version: ['11.8', '12.1']
54
55 steps:
CMakeLists.txt
+1 -1
@@ -31,7 +31,7 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx11
31 # requirements.txt files and should be kept consistent. The ROCm torch
32 # versions are derived from Dockerfile.rocm
33 #
34 -set(TORCH_SUPPORTED_VERSION_CUDA "2.1.2")
34 +set(TORCH_SUPPORTED_VERSION_CUDA "2.2.1")
35 set(TORCH_SUPPORTED_VERSION_ROCM_5X "2.0.1")
36 set(TORCH_SUPPORTED_VERSION_ROCM_6X "2.1.1")
37
Dockerfile
+7 -3
@@ -24,6 +24,13 @@ RUN --mount=type=cache,target=/root/.cache/pip \
24 COPY requirements-dev.txt requirements-dev.txt
25 RUN --mount=type=cache,target=/root/.cache/pip \
26 pip install -r requirements-dev.txt
27 +
28 +# cuda arch list used by torch
29 +# can be useful for both `dev` and `test`
30 +# explicitly set the list to avoid issues with torch 2.2
31 +# see https://github.com/pytorch/pytorch/pull/123243
32 +ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX'
33 +ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
34 #################### BASE BUILD IMAGE ####################
35
36
@@ -47,9 +54,6 @@ COPY requirements.txt requirements.txt
54 COPY pyproject.toml pyproject.toml
55 COPY vllm/__init__.py vllm/__init__.py
56
50 -# cuda arch list used by torch
51 -ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX'
52 -ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
57 # max jobs used by Ninja to build extensions
58 ARG max_jobs=2
59 ENV MAX_JOBS=${max_jobs}
pyproject.toml
+1 -1
@@ -5,7 +5,7 @@ requires = [
5 "ninja",
6 "packaging",
7 "setuptools >= 49.4.0",
8 - "torch == 2.1.2",
8 + "torch == 2.2.1",
9 "wheel",
10 ]
11 build-backend = "setuptools.build_meta"
requirements-build.txt
+1 -1
@@ -3,5 +3,5 @@ cmake>=3.21
3 ninja
4 packaging
5 setuptools>=49.4.0
6 -torch==2.1.2
6 +torch==2.2.1
7 wheel
requirements.txt
+3 -2
@@ -4,11 +4,11 @@ psutil
4 ray >= 2.9
5 sentencepiece # Required for LLaMA tokenizer.
6 numpy
7 -torch == 2.1.2
7 +torch == 2.2.1
8 requests
9 py-cpuinfo
10 transformers >= 4.39.1 # Required for StarCoder2 & Llava.
11 -xformers == 0.0.23.post1 # Required for CUDA 12.1.
11 +xformers == 0.0.25 # Requires PyTorch 2.2.1.
12 fastapi
13 uvicorn[standard]
14 pydantic >= 2.0 # Required for OpenAI server.
@@ -17,3 +17,4 @@ pynvml == 11.5.0
17 triton >= 2.1.0
18 outlines == 0.0.34
19 tiktoken == 0.6.0 # Required for DBRX tokenizer
20 +vllm-nccl-cu12>=2.18<2.19 # for downloading nccl library
setup.py
+10
@@ -328,6 +328,16 @@ def get_requirements() -> List[str]:
328 if _is_cuda():
329 with open(get_path("requirements.txt")) as f:
330 requirements = f.read().strip().split("\n")
331 + cuda_major = torch.version.cuda.split(".")[0]
332 + modified_requirements = []
333 + for req in requirements:
334 + if "vllm-nccl-cu12" in req:
335 + modified_requirements.append(
336 + req.replace("vllm-nccl-cu12",
337 + f"vllm-nccl-cu{cuda_major}"))
338 + else:
339 + modified_requirements.append(req)
340 + requirements = modified_requirements
341 elif _is_hip():
342 with open(get_path("requirements-rocm.txt")) as f:
343 requirements = f.read().strip().split("\n")
vllm/model_executor/parallel_utils/pynccl.py
+12 -2
@@ -21,6 +21,7 @@
21
22 import ctypes
23 import datetime
24 +import glob
25 import os
26
27 # ===================== import region =====================
@@ -34,18 +35,27 @@ logger = init_logger(__name__)
35
36 so_file = os.environ.get("VLLM_NCCL_SO_PATH", "")
37
38 +# check if we have vllm-managed nccl
39 +vllm_nccl_path = None
40 +if torch.version.cuda is not None:
41 + cuda_major = torch.version.cuda.split(".")[0]
42 + path = os.path.expanduser(
43 + f"~/.config/vllm/nccl/cu{cuda_major}/libnccl.so.*")
44 + files = glob.glob(path)
45 + vllm_nccl_path = files[0] if files else None
46 +
47 # manually load the nccl library
48 if so_file:
49 logger.info(
50 f"Loading nccl from environment variable VLLM_NCCL_SO_PATH={so_file}")
51 else:
52 if torch.version.cuda is not None:
43 - so_file = "libnccl.so.2"
53 + so_file = vllm_nccl_path or "libnccl.so.2"
54 elif torch.version.hip is not None:
55 so_file = "librccl.so.1"
56 else:
57 raise ValueError("NCCL only supports CUDA and ROCm backends.")
48 - logger.debug(f"Loading nccl from library {so_file}")
58 + logger.info(f"Loading nccl from library {so_file}")
59
60 try:
61 nccl = ctypes.CDLL(so_file)