A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Rename servers and change port numbers to reduce confusion (#149)
Zhuohan Li committed
Jun 17, 2023 at 00:13 UTC
eedb46bf03818796536358f3767ee2b6a619b4f5
10 files changed
+41
-37
benchmarks/benchmark_async_llm_server.py
+1
-1
@@ -52,7 +52,7 @@ def main(args: argparse.Namespace):
52
if __name__ == "__main__":
53
parser = argparse.ArgumentParser()
54
parser.add_argument("--host", type=str, default="localhost")
55
- parser.add_argument("--port", type=int, default=8001)
55
+ parser.add_argument("--port", type=int, default=8000)
56
parser.add_argument("--max-tokens", type=int, default=128)
57
parser.add_argument("--n-threads", type=int, default=128)
58
args = parser.parse_args()
cacheflow/__init__.py
+2
-2
@@ -2,7 +2,7 @@ from cacheflow.entrypoints.llm import LLM
2
from cacheflow.outputs import RequestOutput, CompletionOutput
3
from cacheflow.sampling_params import SamplingParams
4
from cacheflow.server.arg_utils import ServerArgs
5
-from cacheflow.server.llm_server import LLMServer
5
+from cacheflow.server.llm_server import LLMEngine
6
from cacheflow.server.ray_utils import initialize_cluster
7
8
__version__ = "0.1.0"
@@ -12,7 +12,7 @@ __all__ = [
12
"SamplingParams",
13
"RequestOutput",
14
"CompletionOutput",
15
- "LLMServer",
15
+ "LLMEngine",
16
"ServerArgs",
17
"initialize_cluster",
18
]
cacheflow/entrypoints/api_server.py
renamed
+4
-4
@@ -8,7 +8,7 @@ import uvicorn
8
9
from cacheflow.sampling_params import SamplingParams
10
from cacheflow.server.arg_utils import AsyncServerArgs
11
-from cacheflow.server.async_llm_server import AsyncLLMServer
11
+from cacheflow.server.async_llm_server import AsyncLLMEngine
12
from cacheflow.utils import random_uuid
13
14
TIMEOUT_KEEP_ALIVE = 5 # seconds.
@@ -18,7 +18,7 @@ app = FastAPI()
18
19
@app.post("/generate")
20
async def generate(request: Request) -> Response:
21
- """ Stream the results of the generation request.
21
+ """Generate completion for the request.
22
23
The request should be a JSON object with the following fields:
24
- prompt: the prompt to use for the generation.
@@ -74,12 +74,12 @@ async def generate(request: Request) -> Response:
74
if __name__ == "__main__":
75
parser = argparse.ArgumentParser()
76
parser.add_argument("--host", type=str, default="localhost")
77
- parser.add_argument("--port", type=int, default=8001)
77
+ parser.add_argument("--port", type=int, default=8000)
78
parser = AsyncServerArgs.add_cli_args(parser)
79
args = parser.parse_args()
80
81
server_args = AsyncServerArgs.from_cli_args(args)
82
- server = AsyncLLMServer.from_server_args(server_args)
82
+ server = AsyncLLMEngine.from_server_args(server_args)
83
84
uvicorn.run(app, host=args.host, port=args.port, log_level="debug",
85
timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
cacheflow/entrypoints/llm.py
+3
-3
@@ -6,7 +6,7 @@ from tqdm import tqdm
6
from cacheflow.outputs import RequestOutput
7
from cacheflow.sampling_params import SamplingParams
8
from cacheflow.server.arg_utils import ServerArgs
9
-from cacheflow.server.llm_server import LLMServer
9
+from cacheflow.server.llm_server import LLMEngine
10
from cacheflow.utils import Counter
11
12
@@ -20,7 +20,7 @@ class LLM:
20
mechanism and efficient memory management.
21
22
NOTE: This class is intended to be used for offline inference. For online
23
- serving, use the `AsyncLLMServer` class instead.
23
+ serving, use the `AsyncLLMEngine` class instead.
24
NOTE: For the comprehensive list of arguments, see `ServerArgs`.
25
26
Args:
@@ -52,7 +52,7 @@ class LLM:
52
seed=seed,
53
**kwargs,
54
)
55
- self.llm_server = LLMServer.from_server_args(server_args)
55
+ self.llm_server = LLMEngine.from_server_args(server_args)
56
self.request_counter = Counter()
57
58
def get_tokenizer(
cacheflow/entrypoints/openai/api_server.py
renamed
+2
-2
@@ -15,7 +15,7 @@ import uvicorn
15
16
from cacheflow.outputs import RequestOutput
17
from cacheflow.server.arg_utils import AsyncServerArgs
18
-from cacheflow.server.async_llm_server import AsyncLLMServer
18
+from cacheflow.server.async_llm_server import AsyncLLMEngine
19
from cacheflow.server.tokenizer_utils import get_tokenizer
20
from cacheflow.logger import init_logger
21
from cacheflow.sampling_params import SamplingParams
@@ -319,7 +319,7 @@ if __name__ == "__main__":
319
served_model = args.served_model_name or args.model
320
321
server_args = AsyncServerArgs.from_cli_args(args)
322
- server = AsyncLLMServer.from_server_args(server_args)
322
+ server = AsyncLLMEngine.from_server_args(server_args)
323
324
# A separate tokenizer to map token IDs to strings.
325
tokenizer = get_tokenizer(args.model)
cacheflow/server/async_llm_server.py
+16
-16
@@ -6,7 +6,7 @@ from cacheflow.logger import init_logger
6
from cacheflow.outputs import RequestOutput
7
from cacheflow.sampling_params import SamplingParams
8
from cacheflow.server.arg_utils import AsyncServerArgs
9
-from cacheflow.server.llm_server import LLMServer
9
+from cacheflow.server.llm_server import LLMEngine
10
from cacheflow.server.ray_utils import ray, initialize_cluster
11
12
logger = init_logger(__name__)
@@ -14,26 +14,26 @@ logger = init_logger(__name__)
14
TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
15
16
17
-class AsyncLLMServer:
18
- """An asynchronous wrapper for LLMServer.
17
+class AsyncLLMEngine:
18
+ """An asynchronous wrapper for LLMEngine.
19
20
- This class is used to wrap the LLMServer class to make it asynchronous. It
20
+ This class is used to wrap the LLMEngine class to make it asynchronous. It
21
uses asyncio to create a background loop that keeps processing incoming
22
- requests. The LLMServer is kicked by the generate method when there
22
+ requests. The LLMEngine is kicked by the generate method when there
23
are requests in the waiting queue. The generate method yields the outputs
24
- from the LLMServer to the caller.
24
+ from the LLMEngine to the caller.
25
26
- NOTE: For the comprehensive list of arguments, see `LLMServer`.
26
+ NOTE: For the comprehensive list of arguments, see `LLMEngine`.
27
28
Args:
29
worker_use_ray: Whether to use Ray for model workers. Required for
30
distributed execution. Should be the same as
31
`parallel_config.worker_use_ray`.
32
- server_use_ray: Whether to make LLMServer a Ray actor. If so, the
32
+ server_use_ray: Whether to make LLMEngine a Ray actor. If so, the
33
async frontend will be executed in a separate process as the
34
model workers.
35
log_requests: Whether to log the requests.
36
- *args, *kwargs: Arguments for LLMServer.
36
+ *args, *kwargs: Arguments for LLMEngine.
37
"""
38
def __init__(self, worker_use_ray: bool, server_use_ray: bool,
39
log_requests: bool = True, *args, **kwargs) -> None:
@@ -41,11 +41,11 @@ class AsyncLLMServer:
41
self.server_use_ray = server_use_ray
42
self.log_requests = log_requests
43
if not self.server_use_ray:
44
- server_class = LLMServer
44
+ server_class = LLMEngine
45
elif self.worker_use_ray:
46
- server_class = ray.remote(num_cpus=0)(LLMServer).remote
46
+ server_class = ray.remote(num_cpus=0)(LLMEngine).remote
47
else:
48
- server_class = ray.remote(num_gpus=1)(LLMServer).remote
48
+ server_class = ray.remote(num_gpus=1)(LLMEngine).remote
49
self.server = server_class(*args, **kwargs)
50
# Request id -> request output.
51
self.request_outputs: Dict[str, RequestOutput] = {}
@@ -85,8 +85,8 @@ class AsyncLLMServer:
85
"""Generate outputs for a request.
86
87
Generate outputs for a request. This method is a coroutine. It adds the
88
- request into the waiting queue of the LLMServer and streams the outputs
89
- from the LLMServer to the caller.
88
+ request into the waiting queue of the LLMEngine and streams the outputs
89
+ from the LLMEngine to the caller.
90
91
Args:
92
prompt: The prompt string. Can be None if prompt_token_ids is
@@ -97,7 +97,7 @@ class AsyncLLMServer:
97
use the tokenizer to convert the prompts to token IDs.
98
99
Yields:
100
- The output `RequestOutput` objects from the LLMServer for the
100
+ The output `RequestOutput` objects from the LLMEngine for the
101
request.
102
"""
103
# Preprocess the request.
@@ -200,7 +200,7 @@ class AsyncLLMServer:
200
self.kicking_request_id = None
201
202
@classmethod
203
- def from_server_args(cls, server_args: AsyncServerArgs) -> "AsyncLLMServer":
203
+ def from_server_args(cls, server_args: AsyncServerArgs) -> "AsyncLLMEngine":
204
"""Creates an async LLM server from the server arguments."""
205
# Create the server configs.
206
server_configs = server_args.create_server_configs()
cacheflow/server/llm_server.py
+3
-3
@@ -18,7 +18,7 @@ from cacheflow.worker.worker import Worker
18
logger = init_logger(__name__)
19
20
21
-class LLMServer:
21
+class LLMEngine:
22
"""An LLM server that receives requests and generates texts.
23
24
This is the main class for the CacheFlow LLM server. It receives requests
@@ -29,7 +29,7 @@ class LLMServer:
29
serving throughput.
30
31
The `LLM` class wraps this class for offline batched inference and the
32
- `AsyncLLMServer` class wraps this class for online serving.
32
+ `AsyncLLMEngine` class wraps this class for online serving.
33
34
NOTE: The config arguments are derived from the `ServerArgs` class. For the
35
comprehensive list of arguments, see `ServerArgs`.
@@ -135,7 +135,7 @@ class LLMServer:
135
self._run_workers("init_cache_engine", cache_config=self.cache_config)
136
137
@classmethod
138
- def from_server_args(cls, server_args: ServerArgs) -> "LLMServer":
138
+ def from_server_args(cls, server_args: ServerArgs) -> "LLMEngine":
139
"""Creates an LLM server from the server arguments."""
140
# Create the server configs.
141
server_configs = server_args.create_server_configs()
examples/api_client.py
renamed
+3
-1
@@ -1,3 +1,5 @@
1
+"""Example Python client for cacheflow.entrypoints.api_server"""
2
+
3
import argparse
4
import json
5
from typing import Iterable, List
@@ -45,7 +47,7 @@ def get_response(response: requests.Response) -> List[str]:
47
if __name__ == "__main__":
48
parser = argparse.ArgumentParser()
49
parser.add_argument("--host", type=str, default="localhost")
48
- parser.add_argument("--port", type=int, default=8001)
50
+ parser.add_argument("--port", type=int, default=8000)
51
parser.add_argument("--n", type=int, default=4)
52
parser.add_argument("--prompt", type=str, default="San Francisco is a")
53
parser.add_argument("--stream", action="store_true")
examples/gradio_webserver.py
+3
-2
@@ -9,6 +9,7 @@ def http_bot(prompt):
9
headers = {"User-Agent": "Cacheflow Client"}
10
pload = {
11
"prompt": prompt,
12
+ "stream": True,
13
"max_tokens": 128,
14
}
15
response = requests.post(args.model_url, headers=headers, json=pload, stream=True)
@@ -34,8 +35,8 @@ def build_demo():
35
if __name__ == "__main__":
36
parser = argparse.ArgumentParser()
37
parser.add_argument("--host", type=str, default="localhost")
37
- parser.add_argument("--port", type=int, default=8002)
38
- parser.add_argument("--model-url", type=str, default="http://localhost:8001/generate")
38
+ parser.add_argument("--port", type=int, default=8001)
39
+ parser.add_argument("--model-url", type=str, default="http://localhost:8000/generate")
40
args = parser.parse_args()
41
42
demo = build_demo()
examples/llmserver_example.py
renamed
+4
-3
@@ -1,12 +1,12 @@
1
import argparse
2
3
-from cacheflow import ServerArgs, LLMServer, SamplingParams
3
+from cacheflow import ServerArgs, LLMEngine, SamplingParams
4
5
6
def main(args: argparse.Namespace):
7
# Parse the CLI argument and initialize the server.
8
server_args = ServerArgs.from_cli_args(args)
9
- server = LLMServer.from_server_args(server_args)
9
+ server = LLMEngine.from_server_args(server_args)
10
11
# Test the following prompts.
12
test_prompts = [
@@ -38,7 +38,8 @@ def main(args: argparse.Namespace):
38
39
40
if __name__ == '__main__':
41
- parser = argparse.ArgumentParser(description='Simple CacheFlow server.')
41
+ parser = argparse.ArgumentParser(
42
+ description='Demo on using the LLMEngine class synchronously')
43
parser = ServerArgs.add_cli_args(parser)
44
args = parser.parse_args()
45
main(args)