A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Misc] Define common requirements (#3841)
Woosuk Kwon committed
Apr 5, 2024 at 00:39 UTC
cfaf49a1673c872d2a06560346efb13695f82f35
11 files changed
+62
-77
.github/workflows/publish.yml
+1
-1
@@ -49,7 +49,7 @@ jobs:
49
matrix:
50
os: ['ubuntu-20.04']
51
python-version: ['3.8', '3.9', '3.10', '3.11']
52
- pytorch-version: ['2.2.1'] # Must be the most recent version that meets requirements.txt.
52
+ pytorch-version: ['2.2.1'] # Must be the most recent version that meets requirements-cuda.txt.
53
cuda-version: ['11.8', '12.1']
54
55
steps:
.github/workflows/scripts/build.sh
+1
-1
@@ -9,7 +9,7 @@ LD_LIBRARY_PATH=${cuda_home}/lib64:$LD_LIBRARY_PATH
9
10
# Install requirements
11
$python_executable -m pip install wheel packaging
12
-$python_executable -m pip install -r requirements.txt
12
+$python_executable -m pip install -r requirements-cuda.txt
13
14
# Limit the number of parallel jobs to avoid OOM
15
export MAX_JOBS=1
CONTRIBUTING.md
-1
@@ -21,7 +21,6 @@ Express your support on Twitter if vLLM aids you, or simply offer your appreciat
21
### Build from source
22
23
```bash
24
-pip install -r requirements.txt
24
pip install -e . # This may take several minutes.
25
```
26
Dockerfile
+5
-3
@@ -17,9 +17,10 @@ RUN ldconfig /usr/local/cuda-12.1/compat/
17
WORKDIR /workspace
18
19
# install build and runtime dependencies
20
-COPY requirements.txt requirements.txt
20
+COPY requirements-common.txt requirements-common.txt
21
+COPY requirements-cuda.txt requirements-cuda.txt
22
RUN --mount=type=cache,target=/root/.cache/pip \
22
- pip install -r requirements.txt
23
+ pip install -r requirements-cuda.txt
24
25
# install development dependencies
26
COPY requirements-dev.txt requirements-dev.txt
@@ -51,7 +52,8 @@ COPY csrc csrc
52
COPY setup.py setup.py
53
COPY cmake cmake
54
COPY CMakeLists.txt CMakeLists.txt
54
-COPY requirements.txt requirements.txt
55
+COPY requirements-common.txt requirements-common.txt
56
+COPY requirements-cuda.txt requirements-cuda.txt
57
COPY pyproject.toml pyproject.toml
58
COPY vllm vllm
59
MANIFEST.in
+2
-1
@@ -1,5 +1,6 @@
1
include LICENSE
2
-include requirements.txt
2
+include requirements-common.txt
3
+include requirements-cuda.txt
4
include CMakeLists.txt
5
6
recursive-include cmake *
requirements-common.txt
renamed
+3
-9
@@ -1,20 +1,14 @@
1
-cmake>=3.21
1
+cmake >= 3.21
2
ninja # For faster builds.
3
psutil
4
-ray >= 2.9
4
sentencepiece # Required for LLaMA tokenizer.
5
numpy
7
-torch == 2.2.1
6
requests
7
py-cpuinfo
8
transformers >= 4.39.1 # Required for StarCoder2 & Llava.
11
-xformers == 0.0.25 # Requires PyTorch 2.2.1.
9
fastapi
10
uvicorn[standard]
11
pydantic >= 2.0 # Required for OpenAI server.
12
prometheus_client >= 0.18.0
16
-pynvml == 11.5.0
17
-triton >= 2.1.0
18
-outlines == 0.0.34
19
-tiktoken == 0.6.0 # Required for DBRX tokenizer
20
-vllm-nccl-cu12>=2.18,<2.19 # for downloading nccl library
13
+tiktoken == 0.6.0 # Required for DBRX tokenizer
14
+outlines == 0.0.34 # Requires torch >= 2.1.0
requirements-cpu.txt
+6
-15
@@ -1,15 +1,6 @@
1
-cmake>=3.21
2
-ninja # For faster builds.
3
-psutil
4
-ray >= 2.9
5
-sentencepiece # Required for LLaMA tokenizer.
6
-numpy
7
-transformers >= 4.38.0 # Required for Gemma.
8
-fastapi
9
-uvicorn[standard]
10
-pydantic >= 2.0 # Required for OpenAI server.
11
-prometheus_client >= 0.18.0
12
-torch == 2.2.1+cpu
13
-triton >= 2.1.0
14
-filelock == 3.13.3
15
-py-cpuinfo
\ No newline at end of file
1
+# Common dependencies
2
+-r requirements-common.txt
3
+
4
+# Dependencies for x86_64 CPUs
5
+torch == 2.2.1+cpu
6
+triton >= 2.1.0 # FIXME(woosuk): This is a hack to avoid import error.
requirements-cuda.txt
new
+10
@@ -0,0 +1,10 @@
1
+# Common dependencies
2
+-r requirements-common.txt
3
+
4
+# Dependencies for NVIDIA GPUs
5
+ray >= 2.9
6
+pynvml == 11.5.0
7
+vllm-nccl-cu12>=2.18,<2.19 # for downloading nccl library
8
+torch == 2.2.1
9
+xformers == 0.0.25 # Requires PyTorch 2.2.1
10
+triton >= 2.1.0
requirements-neuron.txt
+4
-9
@@ -1,12 +1,7 @@
1
-sentencepiece # Required for LLaMA tokenizer.
2
-numpy
1
+# Common dependencies
2
+-r requirements-common.txt
3
+
4
+# Dependencies for Neuron devices
5
transformers-neuronx >= 0.9.0
6
torch-neuronx >= 2.1.0
7
neuronx-cc
6
-fastapi
7
-uvicorn[standard]
8
-pydantic >= 2.0 # Required for OpenAI server.
9
-prometheus_client >= 0.18.0
10
-requests
11
-psutil
12
-py-cpuinfo
\ No newline at end of file
requirements-rocm.txt
+4
-17
@@ -1,18 +1,5 @@
1
-cmake>=3.21
2
-ninja # For faster builds.
3
-typing-extensions>=4.8.0
4
-starlette
5
-requests
6
-py-cpuinfo
7
-psutil
1
+# Common dependencies
2
+-r requirements-common.txt
3
+
4
+# Dependencies for AMD GPUs
5
ray == 2.9.3
9
-sentencepiece # Required for LLaMA tokenizer.
10
-numpy
11
-tokenizers>=0.15.0
12
-transformers >= 4.39.1 # Required for StarCoder2 & Llava.
13
-fastapi
14
-uvicorn[standard]
15
-pydantic >= 2.0 # Required for OpenAI server.
16
-prometheus_client >= 0.18.0
17
-outlines == 0.0.34
18
-tiktoken == 0.6.0 # Required for DBRX tokenizer
setup.py
+26
-20
@@ -325,32 +325,38 @@ def read_readme() -> str:
325
326
def get_requirements() -> List[str]:
327
"""Get Python package dependencies from requirements.txt."""
328
- if _is_cuda():
329
- with open(get_path("requirements.txt")) as f:
328
+
329
+ def _read_requirements(filename: str) -> List[str]:
330
+ with open(get_path(filename)) as f:
331
requirements = f.read().strip().split("\n")
331
- cuda_major = torch.version.cuda.split(".")[0]
332
- modified_requirements = []
333
- for req in requirements:
334
- if "vllm-nccl-cu12" in req:
335
- modified_requirements.append(
336
- req.replace("vllm-nccl-cu12",
337
- f"vllm-nccl-cu{cuda_major}"))
338
- else:
339
- modified_requirements.append(req)
340
- requirements = modified_requirements
332
+ resolved_requirements = []
333
+ for line in requirements:
334
+ if line.startswith("-r "):
335
+ resolved_requirements += _read_requirements(line.split()[1])
336
+ else:
337
+ resolved_requirements.append(line)
338
+ return resolved_requirements
339
+
340
+ if _is_cuda():
341
+ requirements = _read_requirements("requirements-cuda.txt")
342
+ cuda_major = torch.version.cuda.split(".")[0]
343
+ modified_requirements = []
344
+ for req in requirements:
345
+ if "vllm-nccl-cu12" in req:
346
+ modified_requirements.append(
347
+ req.replace("vllm-nccl-cu12", f"vllm-nccl-cu{cuda_major}"))
348
+ else:
349
+ modified_requirements.append(req)
350
+ requirements = modified_requirements
351
elif _is_hip():
342
- with open(get_path("requirements-rocm.txt")) as f:
343
- requirements = f.read().strip().split("\n")
352
+ requirements = _read_requirements("requirements-rocm.txt")
353
elif _is_neuron():
345
- with open(get_path("requirements-neuron.txt")) as f:
346
- requirements = f.read().strip().split("\n")
354
+ requirements = _read_requirements("requirements-neuron.txt")
355
elif _is_cpu():
348
- with open(get_path("requirements-cpu.txt")) as f:
349
- requirements = f.read().strip().split("\n")
356
+ requirements = _read_requirements("requirements-cpu.txt")
357
else:
358
raise ValueError(
352
- "Unsupported platform, please use CUDA, ROCM or Neuron.")
353
-
359
+ "Unsupported platform, please use CUDA, ROCm, Neuron, or CPU.")
360
return requirements
361
362