A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
[PyPI] Packaging for PyPI distribution (#140)
Woosuk Kwon committed
Jun 5, 2023 at 20:03 UTC
376725ce74d2d75490eed1840b41de00c0e4acd6
12 files changed
+72
-7
.gitignore
+1
@@ -7,6 +7,7 @@
7
*.csv
8
build/
9
docs/build/
10
+dist/
11
12
*.pkl
13
*.png
MANIFEST.in
new
+4
@@ -0,0 +1,4 @@
1
+include LICENSE
2
+include requirements.txt
3
+
4
+recursive-include csrc *
cacheflow/__init__.py
+2
@@ -5,6 +5,8 @@ from cacheflow.server.arg_utils import ServerArgs
5
from cacheflow.server.llm_server import LLMServer
6
from cacheflow.server.ray_utils import initialize_cluster
7
8
+__version__ = "0.1.0"
9
+
10
__all__ = [
11
"LLM",
12
"SamplingParams",
cacheflow/core/__init__.py
cacheflow/entrypoints/__init__.py
cacheflow/entrypoints/openai/__init__.py
cacheflow/model_executor/layers/__init__.py
cacheflow/server/__init__.py
cacheflow/worker/__init__.py
docs/source/getting_started/installation.rst
+1
-2
@@ -32,7 +32,7 @@ You can install CacheFlow using pip:
32
$ conda activate myenv
33
34
$ # Install CacheFlow.
35
- $ pip install cacheflow
35
+ $ pip install cacheflow # This may take 5-10 minutes.
36
37
38
.. _build_from_source:
@@ -46,5 +46,4 @@ You can also build and install CacheFlow from source.
46
47
$ git clone https://github.com/WoosukKwon/cacheflow.git
48
$ cd cacheflow
49
- $ pip install -r requirements.txt
49
$ pip install -e . # This may take 5-10 minutes.
pyproject.toml
new
+9
@@ -0,0 +1,9 @@
1
+[build-system]
2
+requires = [
3
+ "ninja",
4
+ "packaging",
5
+ "setuptools",
6
+ "torch >= 2.0.0",
7
+ "wheel",
8
+]
9
+build-backend = "setuptools.build_meta"
setup.py
+55
-5
@@ -1,17 +1,24 @@
1
+import io
2
+import os
3
+import re
4
import subprocess
5
from typing import List, Set
6
7
from packaging.version import parse, Version
8
import setuptools
9
import torch
7
-from torch.utils.cpp_extension import BuildExtension, CUDAExtension
8
-from torch.utils.cpp_extension import CUDA_HOME
10
+from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME
11
+
12
+ROOT_DIR = os.path.dirname(__file__)
13
14
# Compiler flags.
11
-CXX_FLAGS = ["-g", "-O2"]
15
+CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
16
# TODO(woosuk): Should we use -O3?
13
-NVCC_FLAGS = ["-O2"]
17
+NVCC_FLAGS = ["-O2", "-std=c++17"]
18
19
+ABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0
20
+CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
21
+NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
22
23
if not torch.cuda.is_available():
24
raise RuntimeError(
@@ -102,15 +109,58 @@ activation_extension = CUDAExtension(
109
ext_modules.append(activation_extension)
110
111
112
+def get_path(*filepath) -> str:
113
+ return os.path.join(ROOT_DIR, *filepath)
114
+
115
+
116
+def find_version(filepath: str):
117
+ """Extract version information from the given filepath.
118
+
119
+ Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py
120
+ """
121
+ with open(filepath) as fp:
122
+ version_match = re.search(
123
+ r"^__version__ = ['\"]([^'\"]*)['\"]", fp.read(), re.M)
124
+ if version_match:
125
+ return version_match.group(1)
126
+ raise RuntimeError("Unable to find version string.")
127
+
128
+
129
+def read_readme() -> str:
130
+ """Read the README file."""
131
+ return io.open(get_path("README.md"), "r", encoding="utf-8").read()
132
+
133
+
134
def get_requirements() -> List[str]:
135
"""Get Python package dependencies from requirements.txt."""
107
- with open("requirements.txt") as f:
136
+ with open(get_path("requirements.txt")) as f:
137
requirements = f.read().strip().split("\n")
138
return requirements
139
140
141
setuptools.setup(
142
name="cacheflow",
143
+ version=find_version(get_path("cacheflow", "__init__.py")),
144
+ author="CacheFlow Team",
145
+ author_email="cacheflow@gmail.com",
146
+ license="Apache 2.0",
147
+ description="CacheFlow: A high-performance LLM Serving System",
148
+ long_description=read_readme(),
149
+ long_description_content_type="text/markdown",
150
+ url="https://github.com/WoosukKwon/cacheflow",
151
+ project_urls={
152
+ "Homepage": "https://github.com/WoosukKwon/cacheflow",
153
+ "Documentation": "https://cacheflow.readthedocs.io/en/latest/",
154
+ },
155
+ classifiers=[
156
+ "Programming Language :: Python :: 3.8",
157
+ "Programming Language :: Python :: 3.9",
158
+ "Programming Language :: Python :: 3.10",
159
+ "License :: OSI Approved :: Apache Software License",
160
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
161
+ ],
162
+ packages=setuptools.find_packages(
163
+ exclude=("benchmarks", "csrc", "docs", "examples", "tests")),
164
python_requires=">=3.8",
165
install_requires=get_requirements(),
166
ext_modules=ext_modules,