A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] manage nccl via a pypi package & upgrade to pt 2.2.1 (#3805)
youkaichao committed
Apr 4, 2024 at 10:26 UTC
ca81ff5196b2fa82b7a9d553cd6e30eab9e72aca
8 files changed
+36
-11
.github/workflows/publish.yml
+1
-1
@@ -49,7 +49,7 @@ jobs:
49
matrix:
50
os: ['ubuntu-20.04']
51
python-version: ['3.8', '3.9', '3.10', '3.11']
52
- pytorch-version: ['2.1.2'] # Must be the most recent version that meets requirements.txt.
52
+ pytorch-version: ['2.2.1'] # Must be the most recent version that meets requirements.txt.
53
cuda-version: ['11.8', '12.1']
54
55
steps:
CMakeLists.txt
+1
-1
@@ -31,7 +31,7 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx11
31
# requirements.txt files and should be kept consistent. The ROCm torch
32
# versions are derived from Dockerfile.rocm
33
#
34
-set(TORCH_SUPPORTED_VERSION_CUDA "2.1.2")
34
+set(TORCH_SUPPORTED_VERSION_CUDA "2.2.1")
35
set(TORCH_SUPPORTED_VERSION_ROCM_5X "2.0.1")
36
set(TORCH_SUPPORTED_VERSION_ROCM_6X "2.1.1")
37
Dockerfile
+7
-3
@@ -24,6 +24,13 @@ RUN --mount=type=cache,target=/root/.cache/pip \
24
COPY requirements-dev.txt requirements-dev.txt
25
RUN --mount=type=cache,target=/root/.cache/pip \
26
pip install -r requirements-dev.txt
27
+
28
+# cuda arch list used by torch
29
+# can be useful for both `dev` and `test`
30
+# explicitly set the list to avoid issues with torch 2.2
31
+# see https://github.com/pytorch/pytorch/pull/123243
32
+ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX'
33
+ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
34
#################### BASE BUILD IMAGE ####################
35
36
@@ -47,9 +54,6 @@ COPY requirements.txt requirements.txt
54
COPY pyproject.toml pyproject.toml
55
COPY vllm/__init__.py vllm/__init__.py
56
50
-# cuda arch list used by torch
51
-ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX'
52
-ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
57
# max jobs used by Ninja to build extensions
58
ARG max_jobs=2
59
ENV MAX_JOBS=${max_jobs}
pyproject.toml
+1
-1
@@ -5,7 +5,7 @@ requires = [
5
"ninja",
6
"packaging",
7
"setuptools >= 49.4.0",
8
- "torch == 2.1.2",
8
+ "torch == 2.2.1",
9
"wheel",
10
]
11
build-backend = "setuptools.build_meta"
requirements-build.txt
+1
-1
@@ -3,5 +3,5 @@ cmake>=3.21
3
ninja
4
packaging
5
setuptools>=49.4.0
6
-torch==2.1.2
6
+torch==2.2.1
7
wheel
requirements.txt
+3
-2
@@ -4,11 +4,11 @@ psutil
4
ray >= 2.9
5
sentencepiece # Required for LLaMA tokenizer.
6
numpy
7
-torch == 2.1.2
7
+torch == 2.2.1
8
requests
9
py-cpuinfo
10
transformers >= 4.39.1 # Required for StarCoder2 & Llava.
11
-xformers == 0.0.23.post1 # Required for CUDA 12.1.
11
+xformers == 0.0.25 # Requires PyTorch 2.2.1.
12
fastapi
13
uvicorn[standard]
14
pydantic >= 2.0 # Required for OpenAI server.
@@ -17,3 +17,4 @@ pynvml == 11.5.0
17
triton >= 2.1.0
18
outlines == 0.0.34
19
tiktoken == 0.6.0 # Required for DBRX tokenizer
20
+vllm-nccl-cu12>=2.18<2.19 # for downloading nccl library
setup.py
+10
@@ -328,6 +328,16 @@ def get_requirements() -> List[str]:
328
if _is_cuda():
329
with open(get_path("requirements.txt")) as f:
330
requirements = f.read().strip().split("\n")
331
+ cuda_major = torch.version.cuda.split(".")[0]
332
+ modified_requirements = []
333
+ for req in requirements:
334
+ if "vllm-nccl-cu12" in req:
335
+ modified_requirements.append(
336
+ req.replace("vllm-nccl-cu12",
337
+ f"vllm-nccl-cu{cuda_major}"))
338
+ else:
339
+ modified_requirements.append(req)
340
+ requirements = modified_requirements
341
elif _is_hip():
342
with open(get_path("requirements-rocm.txt")) as f:
343
requirements = f.read().strip().split("\n")
vllm/model_executor/parallel_utils/pynccl.py
+12
-2
@@ -21,6 +21,7 @@
21
22
import ctypes
23
import datetime
24
+import glob
25
import os
26
27
# ===================== import region =====================
@@ -34,18 +35,27 @@ logger = init_logger(__name__)
35
36
so_file = os.environ.get("VLLM_NCCL_SO_PATH", "")
37
38
+# check if we have vllm-managed nccl
39
+vllm_nccl_path = None
40
+if torch.version.cuda is not None:
41
+ cuda_major = torch.version.cuda.split(".")[0]
42
+ path = os.path.expanduser(
43
+ f"~/.config/vllm/nccl/cu{cuda_major}/libnccl.so.*")
44
+ files = glob.glob(path)
45
+ vllm_nccl_path = files[0] if files else None
46
+
47
# manually load the nccl library
48
if so_file:
49
logger.info(
50
f"Loading nccl from environment variable VLLM_NCCL_SO_PATH={so_file}")
51
else:
52
if torch.version.cuda is not None:
43
- so_file = "libnccl.so.2"
53
+ so_file = vllm_nccl_path or "libnccl.so.2"
54
elif torch.version.hip is not None:
55
so_file = "librccl.so.1"
56
else:
57
raise ValueError("NCCL only supports CUDA and ROCm backends.")
48
- logger.debug(f"Loading nccl from library {so_file}")
58
+ logger.info(f"Loading nccl from library {so_file}")
59
60
try:
61
nccl = ctypes.CDLL(so_file)