@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eedb46bf03
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Rename servers and change port numbers to reduce confusion (#149)

Zhuohan Li committed Jun 17, 2023 at 00:13 UTC eedb46bf03818796536358f3767ee2b6a619b4f5
10 files changed +41 -37
benchmarks/benchmark_async_llm_server.py
+1 -1
@@ -52,7 +52,7 @@ def main(args: argparse.Namespace):
52 if __name__ == "__main__":
53 parser = argparse.ArgumentParser()
54 parser.add_argument("--host", type=str, default="localhost")
55 - parser.add_argument("--port", type=int, default=8001)
55 + parser.add_argument("--port", type=int, default=8000)
56 parser.add_argument("--max-tokens", type=int, default=128)
57 parser.add_argument("--n-threads", type=int, default=128)
58 args = parser.parse_args()
cacheflow/__init__.py
+2 -2
@@ -2,7 +2,7 @@ from cacheflow.entrypoints.llm import LLM
2 from cacheflow.outputs import RequestOutput, CompletionOutput
3 from cacheflow.sampling_params import SamplingParams
4 from cacheflow.server.arg_utils import ServerArgs
5 -from cacheflow.server.llm_server import LLMServer
5 +from cacheflow.server.llm_server import LLMEngine
6 from cacheflow.server.ray_utils import initialize_cluster
7
8 __version__ = "0.1.0"
@@ -12,7 +12,7 @@ __all__ = [
12 "SamplingParams",
13 "RequestOutput",
14 "CompletionOutput",
15 - "LLMServer",
15 + "LLMEngine",
16 "ServerArgs",
17 "initialize_cluster",
18 ]
cacheflow/entrypoints/api_server.py renamed
+4 -4
@@ -8,7 +8,7 @@ import uvicorn
8
9 from cacheflow.sampling_params import SamplingParams
10 from cacheflow.server.arg_utils import AsyncServerArgs
11 -from cacheflow.server.async_llm_server import AsyncLLMServer
11 +from cacheflow.server.async_llm_server import AsyncLLMEngine
12 from cacheflow.utils import random_uuid
13
14 TIMEOUT_KEEP_ALIVE = 5 # seconds.
@@ -18,7 +18,7 @@ app = FastAPI()
18
19 @app.post("/generate")
20 async def generate(request: Request) -> Response:
21 - """ Stream the results of the generation request.
21 + """Generate completion for the request.
22
23 The request should be a JSON object with the following fields:
24 - prompt: the prompt to use for the generation.
@@ -74,12 +74,12 @@ async def generate(request: Request) -> Response:
74 if __name__ == "__main__":
75 parser = argparse.ArgumentParser()
76 parser.add_argument("--host", type=str, default="localhost")
77 - parser.add_argument("--port", type=int, default=8001)
77 + parser.add_argument("--port", type=int, default=8000)
78 parser = AsyncServerArgs.add_cli_args(parser)
79 args = parser.parse_args()
80
81 server_args = AsyncServerArgs.from_cli_args(args)
82 - server = AsyncLLMServer.from_server_args(server_args)
82 + server = AsyncLLMEngine.from_server_args(server_args)
83
84 uvicorn.run(app, host=args.host, port=args.port, log_level="debug",
85 timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
cacheflow/entrypoints/llm.py
+3 -3
@@ -6,7 +6,7 @@ from tqdm import tqdm
6 from cacheflow.outputs import RequestOutput
7 from cacheflow.sampling_params import SamplingParams
8 from cacheflow.server.arg_utils import ServerArgs
9 -from cacheflow.server.llm_server import LLMServer
9 +from cacheflow.server.llm_server import LLMEngine
10 from cacheflow.utils import Counter
11
12
@@ -20,7 +20,7 @@ class LLM:
20 mechanism and efficient memory management.
21
22 NOTE: This class is intended to be used for offline inference. For online
23 - serving, use the `AsyncLLMServer` class instead.
23 + serving, use the `AsyncLLMEngine` class instead.
24 NOTE: For the comprehensive list of arguments, see `ServerArgs`.
25
26 Args:
@@ -52,7 +52,7 @@ class LLM:
52 seed=seed,
53 **kwargs,
54 )
55 - self.llm_server = LLMServer.from_server_args(server_args)
55 + self.llm_server = LLMEngine.from_server_args(server_args)
56 self.request_counter = Counter()
57
58 def get_tokenizer(
cacheflow/entrypoints/openai/api_server.py renamed
+2 -2
@@ -15,7 +15,7 @@ import uvicorn
15
16 from cacheflow.outputs import RequestOutput
17 from cacheflow.server.arg_utils import AsyncServerArgs
18 -from cacheflow.server.async_llm_server import AsyncLLMServer
18 +from cacheflow.server.async_llm_server import AsyncLLMEngine
19 from cacheflow.server.tokenizer_utils import get_tokenizer
20 from cacheflow.logger import init_logger
21 from cacheflow.sampling_params import SamplingParams
@@ -319,7 +319,7 @@ if __name__ == "__main__":
319 served_model = args.served_model_name or args.model
320
321 server_args = AsyncServerArgs.from_cli_args(args)
322 - server = AsyncLLMServer.from_server_args(server_args)
322 + server = AsyncLLMEngine.from_server_args(server_args)
323
324 # A separate tokenizer to map token IDs to strings.
325 tokenizer = get_tokenizer(args.model)
cacheflow/server/async_llm_server.py
+16 -16
@@ -6,7 +6,7 @@ from cacheflow.logger import init_logger
6 from cacheflow.outputs import RequestOutput
7 from cacheflow.sampling_params import SamplingParams
8 from cacheflow.server.arg_utils import AsyncServerArgs
9 -from cacheflow.server.llm_server import LLMServer
9 +from cacheflow.server.llm_server import LLMEngine
10 from cacheflow.server.ray_utils import ray, initialize_cluster
11
12 logger = init_logger(__name__)
@@ -14,26 +14,26 @@ logger = init_logger(__name__)
14 TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
15
16
17 -class AsyncLLMServer:
18 - """An asynchronous wrapper for LLMServer.
17 +class AsyncLLMEngine:
18 + """An asynchronous wrapper for LLMEngine.
19
20 - This class is used to wrap the LLMServer class to make it asynchronous. It
20 + This class is used to wrap the LLMEngine class to make it asynchronous. It
21 uses asyncio to create a background loop that keeps processing incoming
22 - requests. The LLMServer is kicked by the generate method when there
22 + requests. The LLMEngine is kicked by the generate method when there
23 are requests in the waiting queue. The generate method yields the outputs
24 - from the LLMServer to the caller.
24 + from the LLMEngine to the caller.
25
26 - NOTE: For the comprehensive list of arguments, see `LLMServer`.
26 + NOTE: For the comprehensive list of arguments, see `LLMEngine`.
27
28 Args:
29 worker_use_ray: Whether to use Ray for model workers. Required for
30 distributed execution. Should be the same as
31 `parallel_config.worker_use_ray`.
32 - server_use_ray: Whether to make LLMServer a Ray actor. If so, the
32 + server_use_ray: Whether to make LLMEngine a Ray actor. If so, the
33 async frontend will be executed in a separate process as the
34 model workers.
35 log_requests: Whether to log the requests.
36 - *args, *kwargs: Arguments for LLMServer.
36 + *args, *kwargs: Arguments for LLMEngine.
37 """
38 def __init__(self, worker_use_ray: bool, server_use_ray: bool,
39 log_requests: bool = True, *args, **kwargs) -> None:
@@ -41,11 +41,11 @@ class AsyncLLMServer:
41 self.server_use_ray = server_use_ray
42 self.log_requests = log_requests
43 if not self.server_use_ray:
44 - server_class = LLMServer
44 + server_class = LLMEngine
45 elif self.worker_use_ray:
46 - server_class = ray.remote(num_cpus=0)(LLMServer).remote
46 + server_class = ray.remote(num_cpus=0)(LLMEngine).remote
47 else:
48 - server_class = ray.remote(num_gpus=1)(LLMServer).remote
48 + server_class = ray.remote(num_gpus=1)(LLMEngine).remote
49 self.server = server_class(*args, **kwargs)
50 # Request id -> request output.
51 self.request_outputs: Dict[str, RequestOutput] = {}
@@ -85,8 +85,8 @@ class AsyncLLMServer:
85 """Generate outputs for a request.
86
87 Generate outputs for a request. This method is a coroutine. It adds the
88 - request into the waiting queue of the LLMServer and streams the outputs
89 - from the LLMServer to the caller.
88 + request into the waiting queue of the LLMEngine and streams the outputs
89 + from the LLMEngine to the caller.
90
91 Args:
92 prompt: The prompt string. Can be None if prompt_token_ids is
@@ -97,7 +97,7 @@ class AsyncLLMServer:
97 use the tokenizer to convert the prompts to token IDs.
98
99 Yields:
100 - The output `RequestOutput` objects from the LLMServer for the
100 + The output `RequestOutput` objects from the LLMEngine for the
101 request.
102 """
103 # Preprocess the request.
@@ -200,7 +200,7 @@ class AsyncLLMServer:
200 self.kicking_request_id = None
201
202 @classmethod
203 - def from_server_args(cls, server_args: AsyncServerArgs) -> "AsyncLLMServer":
203 + def from_server_args(cls, server_args: AsyncServerArgs) -> "AsyncLLMEngine":
204 """Creates an async LLM server from the server arguments."""
205 # Create the server configs.
206 server_configs = server_args.create_server_configs()
cacheflow/server/llm_server.py
+3 -3
@@ -18,7 +18,7 @@ from cacheflow.worker.worker import Worker
18 logger = init_logger(__name__)
19
20
21 -class LLMServer:
21 +class LLMEngine:
22 """An LLM server that receives requests and generates texts.
23
24 This is the main class for the CacheFlow LLM server. It receives requests
@@ -29,7 +29,7 @@ class LLMServer:
29 serving throughput.
30
31 The `LLM` class wraps this class for offline batched inference and the
32 - `AsyncLLMServer` class wraps this class for online serving.
32 + `AsyncLLMEngine` class wraps this class for online serving.
33
34 NOTE: The config arguments are derived from the `ServerArgs` class. For the
35 comprehensive list of arguments, see `ServerArgs`.
@@ -135,7 +135,7 @@ class LLMServer:
135 self._run_workers("init_cache_engine", cache_config=self.cache_config)
136
137 @classmethod
138 - def from_server_args(cls, server_args: ServerArgs) -> "LLMServer":
138 + def from_server_args(cls, server_args: ServerArgs) -> "LLMEngine":
139 """Creates an LLM server from the server arguments."""
140 # Create the server configs.
141 server_configs = server_args.create_server_configs()
examples/api_client.py renamed
+3 -1
@@ -1,3 +1,5 @@
1 +"""Example Python client for cacheflow.entrypoints.api_server"""
2 +
3 import argparse
4 import json
5 from typing import Iterable, List
@@ -45,7 +47,7 @@ def get_response(response: requests.Response) -> List[str]:
47 if __name__ == "__main__":
48 parser = argparse.ArgumentParser()
49 parser.add_argument("--host", type=str, default="localhost")
48 - parser.add_argument("--port", type=int, default=8001)
50 + parser.add_argument("--port", type=int, default=8000)
51 parser.add_argument("--n", type=int, default=4)
52 parser.add_argument("--prompt", type=str, default="San Francisco is a")
53 parser.add_argument("--stream", action="store_true")
examples/gradio_webserver.py
+3 -2
@@ -9,6 +9,7 @@ def http_bot(prompt):
9 headers = {"User-Agent": "Cacheflow Client"}
10 pload = {
11 "prompt": prompt,
12 + "stream": True,
13 "max_tokens": 128,
14 }
15 response = requests.post(args.model_url, headers=headers, json=pload, stream=True)
@@ -34,8 +35,8 @@ def build_demo():
35 if __name__ == "__main__":
36 parser = argparse.ArgumentParser()
37 parser.add_argument("--host", type=str, default="localhost")
37 - parser.add_argument("--port", type=int, default=8002)
38 - parser.add_argument("--model-url", type=str, default="http://localhost:8001/generate")
38 + parser.add_argument("--port", type=int, default=8001)
39 + parser.add_argument("--model-url", type=str, default="http://localhost:8000/generate")
40 args = parser.parse_args()
41
42 demo = build_demo()
examples/llmserver_example.py renamed
+4 -3
@@ -1,12 +1,12 @@
1 import argparse
2
3 -from cacheflow import ServerArgs, LLMServer, SamplingParams
3 +from cacheflow import ServerArgs, LLMEngine, SamplingParams
4
5
6 def main(args: argparse.Namespace):
7 # Parse the CLI argument and initialize the server.
8 server_args = ServerArgs.from_cli_args(args)
9 - server = LLMServer.from_server_args(server_args)
9 + server = LLMEngine.from_server_args(server_args)
10
11 # Test the following prompts.
12 test_prompts = [
@@ -38,7 +38,8 @@ def main(args: argparse.Namespace):
38
39
40 if __name__ == '__main__':
41 - parser = argparse.ArgumentParser(description='Simple CacheFlow server.')
41 + parser = argparse.ArgumentParser(
42 + description='Demo on using the LLMEngine class synchronously')
43 parser = ServerArgs.add_cli_args(parser)
44 args = parser.parse_args()
45 main(args)