@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 376725ce74
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

[PyPI] Packaging for PyPI distribution (#140)

Woosuk Kwon committed Jun 5, 2023 at 20:03 UTC 376725ce74d2d75490eed1840b41de00c0e4acd6
12 files changed +72 -7
.gitignore
+1
@@ -7,6 +7,7 @@
7 *.csv
8 build/
9 docs/build/
10 +dist/
11
12 *.pkl
13 *.png
MANIFEST.in new
+4
@@ -0,0 +1,4 @@
1 +include LICENSE
2 +include requirements.txt
3 +
4 +recursive-include csrc *
cacheflow/__init__.py
+2
@@ -5,6 +5,8 @@ from cacheflow.server.arg_utils import ServerArgs
5 from cacheflow.server.llm_server import LLMServer
6 from cacheflow.server.ray_utils import initialize_cluster
7
8 +__version__ = "0.1.0"
9 +
10 __all__ = [
11 "LLM",
12 "SamplingParams",
cacheflow/core/__init__.py
cacheflow/entrypoints/__init__.py
cacheflow/entrypoints/openai/__init__.py
cacheflow/model_executor/layers/__init__.py
cacheflow/server/__init__.py
cacheflow/worker/__init__.py
docs/source/getting_started/installation.rst
+1 -2
@@ -32,7 +32,7 @@ You can install CacheFlow using pip:
32 $ conda activate myenv
33
34 $ # Install CacheFlow.
35 - $ pip install cacheflow
35 + $ pip install cacheflow # This may take 5-10 minutes.
36
37
38 .. _build_from_source:
@@ -46,5 +46,4 @@ You can also build and install CacheFlow from source.
46
47 $ git clone https://github.com/WoosukKwon/cacheflow.git
48 $ cd cacheflow
49 - $ pip install -r requirements.txt
49 $ pip install -e . # This may take 5-10 minutes.
pyproject.toml new
+9
@@ -0,0 +1,9 @@
1 +[build-system]
2 +requires = [
3 + "ninja",
4 + "packaging",
5 + "setuptools",
6 + "torch >= 2.0.0",
7 + "wheel",
8 +]
9 +build-backend = "setuptools.build_meta"
setup.py
+55 -5
@@ -1,17 +1,24 @@
1 +import io
2 +import os
3 +import re
4 import subprocess
5 from typing import List, Set
6
7 from packaging.version import parse, Version
8 import setuptools
9 import torch
7 -from torch.utils.cpp_extension import BuildExtension, CUDAExtension
8 -from torch.utils.cpp_extension import CUDA_HOME
10 +from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME
11 +
12 +ROOT_DIR = os.path.dirname(__file__)
13
14 # Compiler flags.
11 -CXX_FLAGS = ["-g", "-O2"]
15 +CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
16 # TODO(woosuk): Should we use -O3?
13 -NVCC_FLAGS = ["-O2"]
17 +NVCC_FLAGS = ["-O2", "-std=c++17"]
18
19 +ABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0
20 +CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
21 +NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
22
23 if not torch.cuda.is_available():
24 raise RuntimeError(
@@ -102,15 +109,58 @@ activation_extension = CUDAExtension(
109 ext_modules.append(activation_extension)
110
111
112 +def get_path(*filepath) -> str:
113 + return os.path.join(ROOT_DIR, *filepath)
114 +
115 +
116 +def find_version(filepath: str):
117 + """Extract version information from the given filepath.
118 +
119 + Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py
120 + """
121 + with open(filepath) as fp:
122 + version_match = re.search(
123 + r"^__version__ = ['\"]([^'\"]*)['\"]", fp.read(), re.M)
124 + if version_match:
125 + return version_match.group(1)
126 + raise RuntimeError("Unable to find version string.")
127 +
128 +
129 +def read_readme() -> str:
130 + """Read the README file."""
131 + return io.open(get_path("README.md"), "r", encoding="utf-8").read()
132 +
133 +
134 def get_requirements() -> List[str]:
135 """Get Python package dependencies from requirements.txt."""
107 - with open("requirements.txt") as f:
136 + with open(get_path("requirements.txt")) as f:
137 requirements = f.read().strip().split("\n")
138 return requirements
139
140
141 setuptools.setup(
142 name="cacheflow",
143 + version=find_version(get_path("cacheflow", "__init__.py")),
144 + author="CacheFlow Team",
145 + author_email="cacheflow@gmail.com",
146 + license="Apache 2.0",
147 + description="CacheFlow: A high-performance LLM Serving System",
148 + long_description=read_readme(),
149 + long_description_content_type="text/markdown",
150 + url="https://github.com/WoosukKwon/cacheflow",
151 + project_urls={
152 + "Homepage": "https://github.com/WoosukKwon/cacheflow",
153 + "Documentation": "https://cacheflow.readthedocs.io/en/latest/",
154 + },
155 + classifiers=[
156 + "Programming Language :: Python :: 3.8",
157 + "Programming Language :: Python :: 3.9",
158 + "Programming Language :: Python :: 3.10",
159 + "License :: OSI Approved :: Apache Software License",
160 + "Topic :: Scientific/Engineering :: Artificial Intelligence",
161 + ],
162 + packages=setuptools.find_packages(
163 + exclude=("benchmarks", "csrc", "docs", "examples", "tests")),
164 python_requires=">=3.8",
165 install_requires=get_requirements(),
166 ext_modules=ext_modules,