@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7addca5935
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Specify python package dependencies in requirements.txt (#78)

Woosuk Kwon committed May 7, 2023 at 16:30 UTC 7addca5935c83806429d7ec557999a505e6f6a35
3 files changed +57 -36
README.md
+3 -8
@@ -1,10 +1,10 @@
1 # CacheFlow
2
3 -## Installation
3 +## Build from source
4
5 ```bash
6 -pip install ninja psutil numpy sentencepiece ray torch transformers xformers
7 -pip install -e .
6 +pip install -r requirements.txt
7 +pip install -e . # This may take several minutes.
8 ```
9
10 ## Test simple server
@@ -21,11 +21,6 @@ python simple_server.py --help
21
22 ## FastAPI server
23
24 -Install the following additional dependencies:
25 -```bash
26 -pip install fastapi uvicorn
27 -```
28 -
24 To start the server:
25 ```bash
26 ray start --head
requirements.txt new
+10
@@ -0,0 +1,10 @@
1 +ninja # For faster builds.
2 +psutil
3 +ray
4 +sentencepiece # Required for LLaMA tokenizer.
5 +numpy
6 +torch >= 2.0.0
7 +transformers >= 4.28.0 # Required for LLaMA.
8 +xformers >= 0.0.19
9 +fastapi
10 +uvicorn
setup.py
+44 -28
@@ -1,14 +1,20 @@
1 +from typing import List
2 +
3 import setuptools
4 import torch
3 -from torch.utils import cpp_extension
5 +from torch.utils.cpp_extension import BuildExtension, CUDAExtension
6 +from torch.utils.cpp_extension import CUDA_HOME
7 +
8
5 -CXX_FLAGS = ['-g']
6 -NVCC_FLAGS = ['-O2']
9 +# Build custom operators.
10 +CXX_FLAGS = ["-g"]
11 +# TODO(woosuk): Should we use -O3?
12 +NVCC_FLAGS = ["-O2"]
13
14 if not torch.cuda.is_available():
15 raise RuntimeError(
10 - f'Cannot find CUDA at CUDA_HOME: {cpp_extension.CUDA_HOME}. '
11 - 'CUDA must be available in order to build the package.')
16 + f"Cannot find CUDA at CUDA_HOME: {CUDA_HOME}. "
17 + "CUDA must be available in order to build the package.")
18
19 # FIXME(woosuk): Consider the case where the machine has multiple GPUs with
20 # different compute capabilities.
@@ -16,52 +22,62 @@ compute_capability = torch.cuda.get_device_capability()
22 major, minor = compute_capability
23 # Enable bfloat16 support if the compute capability is >= 8.0.
24 if major >= 8:
19 - NVCC_FLAGS.append('-DENABLE_BF16')
25 + NVCC_FLAGS.append("-DENABLE_BF16")
26
27 ext_modules = []
28
29 # Cache operations.
24 -cache_extension = cpp_extension.CUDAExtension(
25 - name='cacheflow.cache_ops',
26 - sources=['csrc/cache.cpp', 'csrc/cache_kernels.cu'],
27 - extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
30 +cache_extension = CUDAExtension(
31 + name="cacheflow.cache_ops",
32 + sources=["csrc/cache.cpp", "csrc/cache_kernels.cu"],
33 + extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
34 )
35 ext_modules.append(cache_extension)
36
37 # Attention kernels.
32 -attention_extension = cpp_extension.CUDAExtension(
33 - name='cacheflow.attention_ops',
34 - sources=['csrc/attention.cpp', 'csrc/attention/attention_kernels.cu'],
35 - extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
38 +attention_extension = CUDAExtension(
39 + name="cacheflow.attention_ops",
40 + sources=["csrc/attention.cpp", "csrc/attention/attention_kernels.cu"],
41 + extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
42 )
43 ext_modules.append(attention_extension)
44
45 # Positional encoding kernels.
40 -positional_encoding_extension = cpp_extension.CUDAExtension(
41 - name='cacheflow.pos_encoding_ops',
42 - sources=['csrc/pos_encoding.cpp', 'csrc/pos_encoding_kernels.cu'],
43 - extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
46 +positional_encoding_extension = CUDAExtension(
47 + name="cacheflow.pos_encoding_ops",
48 + sources=["csrc/pos_encoding.cpp", "csrc/pos_encoding_kernels.cu"],
49 + extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
50 )
51 ext_modules.append(positional_encoding_extension)
52
53 # Layer normalization kernels.
48 -layernorm_extension = cpp_extension.CUDAExtension(
49 - name='cacheflow.layernorm_ops',
50 - sources=['csrc/layernorm.cpp', 'csrc/layernorm_kernels.cu'],
51 - extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
54 +layernorm_extension = CUDAExtension(
55 + name="cacheflow.layernorm_ops",
56 + sources=["csrc/layernorm.cpp", "csrc/layernorm_kernels.cu"],
57 + extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
58 )
59 ext_modules.append(layernorm_extension)
60
61 # Activation kernels.
56 -activation_extension = cpp_extension.CUDAExtension(
57 - name='cacheflow.activation_ops',
58 - sources=['csrc/activation.cpp', 'csrc/activation_kernels.cu'],
59 - extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
62 +activation_extension = CUDAExtension(
63 + name="cacheflow.activation_ops",
64 + sources=["csrc/activation.cpp", "csrc/activation_kernels.cu"],
65 + extra_compile_args={"cxx": CXX_FLAGS, "nvcc": NVCC_FLAGS},
66 )
67 ext_modules.append(activation_extension)
68
69 +
70 +def get_requirements() -> List[str]:
71 + """Get Python package dependencies from requirements.txt."""
72 + with open("requirements.txt") as f:
73 + requirements = f.read().strip().split("\n")
74 + return requirements
75 +
76 +
77 setuptools.setup(
64 - name='cacheflow',
78 + name="cacheflow",
79 + python_requires=">=3.8",
80 + install_requires=get_requirements(),
81 ext_modules=ext_modules,
66 - cmdclass={'build_ext': cpp_extension.BuildExtension},
82 + cmdclass={"build_ext": BuildExtension},
83 )