A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Rename servers to engines (#152)
Zhuohan Li committed
Jun 17, 2023 at 17:25 UTC
e5464ee484450c2671dd0226516c99c60ce70d9d
15 files changed
+165
-174
benchmarks/benchmark_latency.py
+1
-1
@@ -14,7 +14,7 @@ def main(args: argparse.Namespace):
14
15
# Process all the requests in a single batch if possible.
16
# NOTE(woosuk): If the request cannot be processed in a single batch,
17
- # the server will automatically process the request in multiple batches.
17
+ # the engine will automatically process the request in multiple batches.
18
llm = LLM(
19
model=args.model,
20
tensor_parallel_size=args.tensor_parallel_size,
benchmarks/benchmark_serving.py
+1
-1
@@ -2,7 +2,7 @@
2
3
On the server side, run one of the following commands:
4
(CacheFlow backend)
5
- python -m cacheflow.entrypoints.simple_fastapi_frontend \
5
+ python -m cacheflow.entrypoints.api_server \
6
--disable-log-requests --model <your_model>
7
8
(TGI backend)
benchmarks/benchmark_throughput.py
+2
-2
@@ -84,7 +84,7 @@ def run_cacheflow(
84
seed=seed,
85
)
86
87
- # Add the requests to the server.
87
+ # Add the requests to the engine.
88
for prompt, _, output_len in requests:
89
sampling_params = SamplingParams(
90
n=n,
@@ -103,7 +103,7 @@ def run_cacheflow(
103
104
start = time.time()
105
# FIXME(woosuk): Do use internal method.
106
- llm._run_server(use_tqdm=True)
106
+ llm._run_engine(use_tqdm=True)
107
end = time.time()
108
return end - start
109
cacheflow/__init__.py
+5
-5
@@ -1,9 +1,9 @@
1
+from cacheflow.engine.arg_utils import EngineArgs
2
+from cacheflow.engine.llm_engine import LLMEngine
3
+from cacheflow.engine.ray_utils import initialize_cluster
4
from cacheflow.entrypoints.llm import LLM
2
-from cacheflow.outputs import RequestOutput, CompletionOutput
5
+from cacheflow.outputs import CompletionOutput, RequestOutput
6
from cacheflow.sampling_params import SamplingParams
4
-from cacheflow.server.arg_utils import ServerArgs
5
-from cacheflow.server.llm_server import LLMEngine
6
-from cacheflow.server.ray_utils import initialize_cluster
7
8
__version__ = "0.1.0"
9
@@ -13,6 +13,6 @@ __all__ = [
13
"RequestOutput",
14
"CompletionOutput",
15
"LLMEngine",
16
- "ServerArgs",
16
+ "EngineArgs",
17
"initialize_cluster",
18
]
cacheflow/core/scheduler.py
+1
-1
@@ -216,7 +216,7 @@ class Scheduler:
216
if not self.log_stats:
217
return scheduler_outputs, prompt_group_ids
218
219
- # TODO(woosuk): Move the below code to server.
219
+ # TODO(woosuk): Move the below code to the engine.
220
now = time.time()
221
if num_batched_tokens > 0:
222
self.num_input_tokens.append((now, num_batched_tokens))
cacheflow/engine/__init__.py
renamed
cacheflow/engine/arg_utils.py
renamed
+24
-24
@@ -8,8 +8,8 @@ from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
8
9
10
@dataclass
11
-class ServerArgs:
12
- """Arguments for CacheFlow servers."""
11
+class EngineArgs:
12
+ """Arguments for CacheFlow engine."""
13
model: str
14
download_dir: Optional[str] = None
15
use_np_weights: bool = False
@@ -33,12 +33,12 @@ class ServerArgs:
33
def add_cli_args(
34
parser: argparse.ArgumentParser,
35
) -> argparse.ArgumentParser:
36
- """Shared CLI arguments for CacheFlow servers."""
36
+ """Shared CLI arguments for CacheFlow engine."""
37
# Model arguments
38
parser.add_argument('--model', type=str, default='facebook/opt-125m',
39
help='name or path of the huggingface model to use')
40
parser.add_argument('--download-dir', type=str,
41
- default=ServerArgs.download_dir,
41
+ default=EngineArgs.download_dir,
42
help='directory to download and load the weights, '
43
'default to the default cache dir of '
44
'huggingface')
@@ -49,7 +49,7 @@ class ServerArgs:
49
parser.add_argument('--use-dummy-weights', action='store_true',
50
help='use dummy values for model weights')
51
# TODO(woosuk): Support FP32.
52
- parser.add_argument('--dtype', type=str, default=ServerArgs.dtype,
52
+ parser.add_argument('--dtype', type=str, default=EngineArgs.dtype,
53
choices=['auto', 'half', 'bfloat16', 'float'],
54
help='data type for model weights and activations. '
55
'The "auto" option will use FP16 precision '
@@ -60,46 +60,46 @@ class ServerArgs:
60
help='use Ray for distributed serving, will be '
61
'automatically set when using more than 1 GPU')
62
parser.add_argument('--pipeline-parallel-size', '-pp', type=int,
63
- default=ServerArgs.pipeline_parallel_size,
63
+ default=EngineArgs.pipeline_parallel_size,
64
help='number of pipeline stages')
65
parser.add_argument('--tensor-parallel-size', '-tp', type=int,
66
- default=ServerArgs.tensor_parallel_size,
66
+ default=EngineArgs.tensor_parallel_size,
67
help='number of tensor parallel replicas')
68
# KV cache arguments
69
parser.add_argument('--block-size', type=int,
70
- default=ServerArgs.block_size,
70
+ default=EngineArgs.block_size,
71
choices=[8, 16, 32],
72
help='token block size')
73
# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
74
- parser.add_argument('--seed', type=int, default=ServerArgs.seed,
74
+ parser.add_argument('--seed', type=int, default=EngineArgs.seed,
75
help='random seed')
76
parser.add_argument('--swap-space', type=int,
77
- default=ServerArgs.swap_space,
77
+ default=EngineArgs.swap_space,
78
help='CPU swap space size (GiB) per GPU')
79
parser.add_argument('--gpu-memory-utilization', type=float,
80
- default=ServerArgs.gpu_memory_utilization,
80
+ default=EngineArgs.gpu_memory_utilization,
81
help='the percentage of GPU memory to be used for'
82
'the model executor')
83
parser.add_argument('--max-num-batched-tokens', type=int,
84
- default=ServerArgs.max_num_batched_tokens,
84
+ default=EngineArgs.max_num_batched_tokens,
85
help='maximum number of batched tokens per '
86
'iteration')
87
parser.add_argument('--max-num-seqs', type=int,
88
- default=ServerArgs.max_num_seqs,
88
+ default=EngineArgs.max_num_seqs,
89
help='maximum number of sequences per iteration')
90
parser.add_argument('--disable-log-stats', action='store_true',
91
help='disable logging statistics')
92
return parser
93
94
@classmethod
95
- def from_cli_args(cls, args: argparse.Namespace) -> "ServerArgs":
95
+ def from_cli_args(cls, args: argparse.Namespace) -> "EngineArgs":
96
# Get the list of attributes of this dataclass.
97
attrs = [attr.name for attr in dataclasses.fields(cls)]
98
# Set the attributes from the parsed arguments.
99
- server_args = cls(**{attr: getattr(args, attr) for attr in attrs})
100
- return server_args
99
+ engine_args = cls(**{attr: getattr(args, attr) for attr in attrs})
100
+ return engine_args
101
102
- def create_server_configs(
102
+ def create_engine_configs(
103
self,
104
) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
105
# Initialize the configs.
@@ -117,19 +117,19 @@ class ServerArgs:
117
118
119
@dataclass
120
-class AsyncServerArgs(ServerArgs):
121
- """Arguments for asynchronous CacheFlow servers."""
122
- server_use_ray: bool = False
120
+class AsyncEngineArgs(EngineArgs):
121
+ """Arguments for asynchronous CacheFlow engine."""
122
+ engine_use_ray: bool = False
123
disable_log_requests: bool = False
124
125
@staticmethod
126
def add_cli_args(
127
parser: argparse.ArgumentParser,
128
) -> argparse.ArgumentParser:
129
- parser = ServerArgs.add_cli_args(parser)
130
- parser.add_argument('--server-use-ray', action='store_true',
131
- help='use Ray to start the LLM server in a '
132
- 'separate process as the web server process.')
129
+ parser = EngineArgs.add_cli_args(parser)
130
+ parser.add_argument('--engine-use-ray', action='store_true',
131
+ help='use Ray to start the LLM engine in a '
132
+ 'separate process as the server process.')
133
parser.add_argument('--disable-log-requests', action='store_true',
134
help='disable logging requests')
135
return parser
cacheflow/engine/async_llm_engine.py
renamed
+53
-53
@@ -2,12 +2,12 @@ import asyncio
2
import time
3
from typing import Dict, List, Optional
4
5
+from cacheflow.engine.arg_utils import AsyncEngineArgs
6
+from cacheflow.engine.llm_engine import LLMEngine
7
+from cacheflow.engine.ray_utils import initialize_cluster, ray
8
from cacheflow.logger import init_logger
9
from cacheflow.outputs import RequestOutput
10
from cacheflow.sampling_params import SamplingParams
8
-from cacheflow.server.arg_utils import AsyncServerArgs
9
-from cacheflow.server.llm_server import LLMEngine
10
-from cacheflow.server.ray_utils import ray, initialize_cluster
11
12
logger = init_logger(__name__)
13
@@ -29,44 +29,44 @@ class AsyncLLMEngine:
29
worker_use_ray: Whether to use Ray for model workers. Required for
30
distributed execution. Should be the same as
31
`parallel_config.worker_use_ray`.
32
- server_use_ray: Whether to make LLMEngine a Ray actor. If so, the
32
+ engine_use_ray: Whether to make LLMEngine a Ray actor. If so, the
33
async frontend will be executed in a separate process as the
34
model workers.
35
log_requests: Whether to log the requests.
36
*args, *kwargs: Arguments for LLMEngine.
37
"""
38
- def __init__(self, worker_use_ray: bool, server_use_ray: bool,
38
+ def __init__(self, worker_use_ray: bool, engine_use_ray: bool,
39
log_requests: bool = True, *args, **kwargs) -> None:
40
self.worker_use_ray = worker_use_ray
41
- self.server_use_ray = server_use_ray
41
+ self.engine_use_ray = engine_use_ray
42
self.log_requests = log_requests
43
- if not self.server_use_ray:
44
- server_class = LLMEngine
43
+ if not self.engine_use_ray:
44
+ engine_class = LLMEngine
45
elif self.worker_use_ray:
46
- server_class = ray.remote(num_cpus=0)(LLMEngine).remote
46
+ engine_class = ray.remote(num_cpus=0)(LLMEngine).remote
47
else:
48
- server_class = ray.remote(num_gpus=1)(LLMEngine).remote
49
- self.server = server_class(*args, **kwargs)
48
+ engine_class = ray.remote(num_gpus=1)(LLMEngine).remote
49
+ self.engine = engine_class(*args, **kwargs)
50
# Request id -> request output.
51
self.request_outputs: Dict[str, RequestOutput] = {}
52
# Request id -> event to notify that there is new output.
53
self.request_events: Dict[str, asyncio.Event] = {}
54
- self.is_server_running = False
54
+ self.is_engine_running = False
55
self.kicking_request_id: Optional[str] = None
56
57
- async def server_step(self, kicking_request_id: Optional[str] = None):
58
- """Kick the server to process the waiting requests."""
59
- self.is_server_running = True
57
+ async def engine_step(self, kicking_request_id: Optional[str] = None):
58
+ """Kick the engine to process the waiting requests."""
59
+ self.is_engine_running = True
60
self.kicking_request_id = kicking_request_id
61
- if self.server_use_ray:
62
- request_outputs = await self.server.step.remote()
61
+ if self.engine_use_ray:
62
+ request_outputs = await self.engine.step.remote()
63
else:
64
# Yield to the event loop to allow other coroutines to run
65
- # while is_server_running is True. This let the server to add new
65
+ # while is_engine_running is True. This let the engine to add new
66
# requests into the queue.
67
await asyncio.sleep(0)
68
- request_outputs = self.server.step()
69
- self.is_server_running = False
68
+ request_outputs = self.engine.step()
69
+ self.is_engine_running = False
70
self.kicking_request_id = None
71
72
# Notify the waiting coroutines that there are new outputs ready.
@@ -104,7 +104,7 @@ class AsyncLLMEngine:
104
arrival_time = time.time()
105
106
# Create an event to notify us that there is new output from the
107
- # cacheflow server.
107
+ # cacheflow engine.
108
request_event = asyncio.Event()
109
self.request_events[request_id] = request_event
110
@@ -114,31 +114,31 @@ class AsyncLLMEngine:
114
f"sampling params: {sampling_params}, "
115
f"prompt token ids: {prompt_token_ids}.")
116
117
- # Add the request into the cacheflow server's waiting queue.
118
- if self.server_use_ray:
119
- await self.server.add_request.remote(
117
+ # Add the request into the cacheflow engine's waiting queue.
118
+ if self.engine_use_ray:
119
+ await self.engine.add_request.remote(
120
request_id, prompt, sampling_params,
121
prompt_token_ids=prompt_token_ids,
122
arrival_time=arrival_time)
123
else:
124
- self.server.add_request(
124
+ self.engine.add_request(
125
request_id, prompt, sampling_params,
126
prompt_token_ids=prompt_token_ids,
127
arrival_time=arrival_time)
128
129
- # The cacheflow server does not have a background loop that keeps
129
+ # The cacheflow engine does not have a background loop that keeps
130
# processing incoming requests. Therefore, we need to keep kicking
131
- # the server to process the requests.
131
+ # the engine to process the requests.
132
while True:
133
if request_id not in self.request_events:
134
# The request has been aborted.
135
return
136
137
- # Kick the server if the server is not running.
138
- if not self.is_server_running:
139
- await self.server_step(request_id)
137
+ # Kick the engine if the engine is not running.
138
+ if not self.is_engine_running:
139
+ await self.engine_step(request_id)
140
141
- # Wait for new output. The group_event will be set in server_step
141
+ # Wait for new output. The group_event will be set in engine_step
142
# when there is new output available for the sequence group.
143
# Added a timeout to prevent deadlock.
144
try:
@@ -160,11 +160,11 @@ class AsyncLLMEngine:
160
161
del self.request_outputs[request_id]
162
del self.request_events[request_id]
163
- # Kick the server if the server is not running. This is to
164
- # prevent that there are still requests in server's waiting
163
+ # Kick the engine if the engine is not running. This is to
164
+ # prevent that there are still requests in engine's waiting
165
# queue to be executed.
166
- if not self.is_server_running:
167
- await self.server_step()
166
+ if not self.is_engine_running:
167
+ await self.engine_step()
168
break
169
170
async def abort(self, request_id: str) -> None:
@@ -183,36 +183,36 @@ class AsyncLLMEngine:
183
if self.log_requests:
184
logger.info(f"Aborted request {request_id}.")
185
186
- if self.server_use_ray:
187
- await self.server.abort_request.remote(request_id)
186
+ if self.engine_use_ray:
187
+ await self.engine.abort_request.remote(request_id)
188
else:
189
- self.server.abort_request(request_id)
189
+ self.engine.abort_request(request_id)
190
191
if request_id in self.request_events:
192
del self.request_events[request_id]
193
if request_id in self.request_outputs:
194
del self.request_outputs[request_id]
195
196
- # To prevent deadlock when a request is aborted while the server is
196
+ # To prevent deadlock when a request is aborted while the engine is
197
# running.
198
if self.kicking_request_id == request_id:
199
- self.is_server_running = False
199
+ self.is_engine_running = False
200
self.kicking_request_id = None
201
202
@classmethod
203
- def from_server_args(cls, server_args: AsyncServerArgs) -> "AsyncLLMEngine":
204
- """Creates an async LLM server from the server arguments."""
205
- # Create the server configs.
206
- server_configs = server_args.create_server_configs()
207
- parallel_config = server_configs[2]
203
+ def from_engine_args(cls, engine_args: AsyncEngineArgs) -> "AsyncLLMEngine":
204
+ """Creates an async LLM engine from the engine arguments."""
205
+ # Create the engine configs.
206
+ engine_configs = engine_args.create_engine_configs()
207
+ parallel_config = engine_configs[2]
208
# Initialize the cluster.
209
distributed_init_method, devices = initialize_cluster(
210
- parallel_config, server_args.server_use_ray)
211
- # Create the LLM server.
212
- server = cls(server_args.worker_use_ray,
213
- server_args.server_use_ray,
214
- not server_args.disable_log_requests,
215
- *server_configs,
210
+ parallel_config, engine_args.engine_use_ray)
211
+ # Create the async LLM engine.
212
+ engine = cls(engine_args.worker_use_ray,
213
+ engine_args.engine_use_ray,
214
+ not engine_args.disable_log_requests,
215
+ *engine_configs,
216
distributed_init_method, devices,
217
- log_stats=not server_args.disable_log_stats)
218
- return server
217
+ log_stats=not engine_args.disable_log_stats)
218
+ return engine
cacheflow/engine/llm_engine.py
renamed
+21
-21
@@ -4,13 +4,13 @@ from typing import Any, List, Optional
4
from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
5
SchedulerConfig)
6
from cacheflow.core.scheduler import Scheduler
7
+from cacheflow.engine.arg_utils import EngineArgs
8
+from cacheflow.engine.ray_utils import DeviceID, initialize_cluster, ray
9
+from cacheflow.engine.tokenizer_utils import (detokenize_incrementally,
10
+ get_tokenizer)
11
from cacheflow.logger import init_logger
12
from cacheflow.outputs import RequestOutput
13
from cacheflow.sampling_params import SamplingParams
10
-from cacheflow.server.arg_utils import ServerArgs
11
-from cacheflow.server.ray_utils import DeviceID, initialize_cluster, ray
12
-from cacheflow.server.tokenizer_utils import (get_tokenizer,
13
- detokenize_incrementally)
14
from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
15
from cacheflow.utils import Counter
16
from cacheflow.worker.worker import Worker
@@ -19,9 +19,9 @@ logger = init_logger(__name__)
19
20
21
class LLMEngine:
22
- """An LLM server that receives requests and generates texts.
22
+ """An LLM engine that receives requests and generates texts.
23
24
- This is the main class for the CacheFlow LLM server. It receives requests
24
+ This is the main class for the CacheFlow LLM engine. It receives requests
25
from clients and generates texts from the LLM. It includes a tokenizer, a
26
language model (possibly distributed across multiple GPUs), and GPU memory
27
space allocated for intermediate states (aka KV cache). This class utilizes
@@ -31,8 +31,8 @@ class LLMEngine:
31
The `LLM` class wraps this class for offline batched inference and the
32
`AsyncLLMEngine` class wraps this class for online serving.
33
34
- NOTE: The config arguments are derived from the `ServerArgs` class. For the
35
- comprehensive list of arguments, see `ServerArgs`.
34
+ NOTE: The config arguments are derived from the `EngineArgs` class. For the
35
+ comprehensive list of arguments, see `EngineArgs`.
36
37
Args:
38
model_config: The configuration related to the LLM model.
@@ -58,7 +58,7 @@ class LLMEngine:
58
log_stats: bool,
59
) -> None:
60
logger.info(
61
- "Initializing an LLM server with config: "
61
+ "Initializing an LLM engine with config: "
62
f"model={model_config.model!r}, "
63
f"dtype={model_config.dtype}, "
64
f"use_dummy_weights={model_config.use_dummy_weights}, "
@@ -135,17 +135,17 @@ class LLMEngine:
135
self._run_workers("init_cache_engine", cache_config=self.cache_config)
136
137
@classmethod
138
- def from_server_args(cls, server_args: ServerArgs) -> "LLMEngine":
139
- """Creates an LLM server from the server arguments."""
140
- # Create the server configs.
141
- server_configs = server_args.create_server_configs()
142
- parallel_config = server_configs[2]
138
+ def from_engine_args(cls, engine_args: EngineArgs) -> "LLMEngine":
139
+ """Creates an LLM engine from the engine arguments."""
140
+ # Create the engine configs.
141
+ engine_configs = engine_args.create_engine_configs()
142
+ parallel_config = engine_configs[2]
143
# Initialize the cluster.
144
distributed_init_method, devices = initialize_cluster(parallel_config)
145
- # Create the LLM server.
146
- server = cls(*server_configs, distributed_init_method, devices,
147
- log_stats=not server_args.disable_log_stats)
148
- return server
145
+ # Create the LLM engine.
146
+ engine = cls(*engine_configs, distributed_init_method, devices,
147
+ log_stats=not engine_args.disable_log_stats)
148
+ return engine
149
150
def add_request(
151
self,
@@ -155,10 +155,10 @@ class LLMEngine:
155
prompt_token_ids: Optional[List[int]] = None,
156
arrival_time: Optional[float] = None,
157
) -> None:
158
- """Add a request to the server's request pool.
158
+ """Add a request to the engine's request pool.
159
160
The request is added to the request pool and will be processed by the
161
- scheduler as `server.step()` is called. The exact scheduling policy is
161
+ scheduler as `engine.step()` is called. The exact scheduling policy is
162
determined by the scheduler.
163
164
Args:
@@ -211,7 +211,7 @@ class LLMEngine:
211
def step(self) -> List[RequestOutput]:
212
"""Performs one decoding iteration and returns newly generated results.
213
214
- This function performs one decoding iteration for the server. It first
214
+ This function performs one decoding iteration of the engine. It first
215
schedules the sequences to be executed in the next iteration and the
216
token blocks to be swapped in/out/copy. Then, it executes the model
217
and updates the scheduler with the model outputs. Finally, it decodes
cacheflow/engine/ray_utils.py
renamed
+6
-6
@@ -13,15 +13,15 @@ DeviceID = Tuple[int, Optional[str], int] # rank, node resource (node IP), devi
13
14
def initialize_cluster(
15
parallel_config: ParallelConfig,
16
- server_use_ray: bool = False,
17
- ray_server_address: Optional[str] = None,
16
+ engine_use_ray: bool = False,
17
+ ray_address: Optional[str] = None,
18
) -> Tuple[str, List[List[DeviceID]]]:
19
"""Initialize the distributed cluster probably with Ray.
20
21
Args:
22
parallel_config: The configurations for parallel execution.
23
- server_use_ray: Whether to use Ray for async server.
24
- ray_server_address: The address of the Ray cluster. If None, uses
23
+ engine_use_ray: Whether to use Ray for async engine.
24
+ ray_address: The address of the Ray cluster. If None, uses
25
the default Ray cluster address.
26
27
Returns:
@@ -31,13 +31,13 @@ def initialize_cluster(
31
each worker in each pipeline stage. Each device ID is a tuple of
32
(rank, node resource, device id).
33
"""
34
- if parallel_config.worker_use_ray or server_use_ray:
34
+ if parallel_config.worker_use_ray or engine_use_ray:
35
if ray is None:
36
raise ImportError(
37
"Ray is not installed. Please install Ray to use distributed "
38
"serving.")
39
# Connect to a ray cluster.
40
- ray.init(address=ray_server_address)
40
+ ray.init(address=ray_address)
41
42
if not parallel_config.worker_use_ray:
43
# Initialize cluster locally.
cacheflow/engine/tokenizer_utils.py
renamed
cacheflow/entrypoints/api_server.py
+8
-8
@@ -6,9 +6,9 @@ from fastapi import BackgroundTasks, FastAPI, Request
6
from fastapi.responses import Response, StreamingResponse
7
import uvicorn
8
9
+from cacheflow.engine.arg_utils import AsyncEngineArgs
10
+from cacheflow.engine.async_llm_engine import AsyncLLMEngine
11
from cacheflow.sampling_params import SamplingParams
10
-from cacheflow.server.arg_utils import AsyncServerArgs
11
-from cacheflow.server.async_llm_server import AsyncLLMEngine
12
from cacheflow.utils import random_uuid
13
14
TIMEOUT_KEEP_ALIVE = 5 # seconds.
@@ -30,7 +30,7 @@ async def generate(request: Request) -> Response:
30
stream = request_dict.pop("stream", False)
31
sampling_params = SamplingParams(**request_dict)
32
request_id = random_uuid()
33
- results_generator = server.generate(prompt, sampling_params, request_id)
33
+ results_generator = engine.generate(prompt, sampling_params, request_id)
34
35
# Streaming case
36
async def stream_results() -> AsyncGenerator[bytes, None]:
@@ -44,7 +44,7 @@ async def generate(request: Request) -> Response:
44
yield (json.dumps(ret) + "\0").encode("utf-8")
45
46
async def abort_request() -> None:
47
- await server.abort(request_id)
47
+ await engine.abort(request_id)
48
49
if stream:
50
background_tasks = BackgroundTasks()
@@ -57,7 +57,7 @@ async def generate(request: Request) -> Response:
57
async for request_output in results_generator:
58
if await request.is_disconnected():
59
# Abort the request if the client disconnects.
60
- await server.abort(request_id)
60
+ await engine.abort(request_id)
61
return Response(status_code=499)
62
final_output = request_output
63
@@ -75,11 +75,11 @@ if __name__ == "__main__":
75
parser = argparse.ArgumentParser()
76
parser.add_argument("--host", type=str, default="localhost")
77
parser.add_argument("--port", type=int, default=8000)
78
- parser = AsyncServerArgs.add_cli_args(parser)
78
+ parser = AsyncEngineArgs.add_cli_args(parser)
79
args = parser.parse_args()
80
81
- server_args = AsyncServerArgs.from_cli_args(args)
82
- server = AsyncLLMEngine.from_server_args(server_args)
81
+ engine_args = AsyncEngineArgs.from_cli_args(args)
82
+ engine = AsyncLLMEngine.from_engine_args(engine_args)
83
84
uvicorn.run(app, host=args.host, port=args.port, log_level="debug",
85
timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
cacheflow/entrypoints/llm.py
+15
-15
@@ -1,12 +1,12 @@
1
from typing import List, Optional, Union
2
3
-from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
3
from tqdm import tqdm
4
+from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
5
6
+from cacheflow.engine.arg_utils import EngineArgs
7
+from cacheflow.engine.llm_engine import LLMEngine
8
from cacheflow.outputs import RequestOutput
9
from cacheflow.sampling_params import SamplingParams
8
-from cacheflow.server.arg_utils import ServerArgs
9
-from cacheflow.server.llm_server import LLMEngine
10
from cacheflow.utils import Counter
11
12
@@ -21,7 +21,7 @@ class LLM:
21
22
NOTE: This class is intended to be used for offline inference. For online
23
serving, use the `AsyncLLMEngine` class instead.
24
- NOTE: For the comprehensive list of arguments, see `ServerArgs`.
24
+ NOTE: For the comprehensive list of arguments, see `EngineArgs`.
25
26
Args:
27
model: The name or path of a HuggingFace Transformers model.
@@ -45,20 +45,20 @@ class LLM:
45
) -> None:
46
if "disable_log_stats" not in kwargs:
47
kwargs["disable_log_stats"] = True
48
- server_args = ServerArgs(
48
+ engine_args = EngineArgs(
49
model=model,
50
tensor_parallel_size=tensor_parallel_size,
51
dtype=dtype,
52
seed=seed,
53
**kwargs,
54
)
55
- self.llm_server = LLMEngine.from_server_args(server_args)
55
+ self.llm_engine = LLMEngine.from_engine_args(engine_args)
56
self.request_counter = Counter()
57
58
def get_tokenizer(
59
self,
60
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
61
- return self.llm_server.tokenizer
61
+ return self.llm_engine.tokenizer
62
63
def generate(
64
self,
@@ -99,7 +99,7 @@ class LLM:
99
# Use default sampling params.
100
sampling_params = SamplingParams()
101
102
- # Add requests to the server.
102
+ # Add requests to the engine.
103
if prompts is not None:
104
num_requests = len(prompts)
105
else:
@@ -111,7 +111,7 @@ class LLM:
111
else:
112
token_ids = prompt_token_ids[i]
113
self._add_request(prompt, sampling_params, token_ids)
114
- return self._run_server(use_tqdm)
114
+ return self._run_engine(use_tqdm)
115
116
def _add_request(
117
self,
@@ -120,18 +120,18 @@ class LLM:
120
prompt_token_ids: Optional[List[int]],
121
) -> None:
122
request_id = str(next(self.request_counter))
123
- self.llm_server.add_request(request_id, prompt, sampling_params,
123
+ self.llm_engine.add_request(request_id, prompt, sampling_params,
124
prompt_token_ids)
125
126
- def _run_server(self, use_tqdm: bool) -> List[RequestOutput]:
126
+ def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:
127
# Initialize tqdm.
128
if use_tqdm:
129
- num_requests = self.llm_server.get_num_unfinished_requests()
129
+ num_requests = self.llm_engine.get_num_unfinished_requests()
130
pbar = tqdm(total=num_requests, desc="Processed prompts")
131
- # Run the server.
131
+ # Run the engine.
132
outputs: List[RequestOutput] = []
133
- while self.llm_server.has_unfinished_requests():
134
- step_outputs = self.llm_server.step()
133
+ while self.llm_engine.has_unfinished_requests():
134
+ step_outputs = self.llm_engine.step()
135
for output in step_outputs:
136
if output.finished():
137
outputs.append(output)
cacheflow/entrypoints/openai/api_server.py
+18
-27
@@ -10,29 +10,20 @@ import fastapi
10
from fastapi import BackgroundTasks, Request
11
from fastapi.exceptions import RequestValidationError
12
from fastapi.middleware.cors import CORSMiddleware
13
-from fastapi.responses import StreamingResponse, JSONResponse
13
+from fastapi.responses import JSONResponse, StreamingResponse
14
import uvicorn
15
16
-from cacheflow.outputs import RequestOutput
17
-from cacheflow.server.arg_utils import AsyncServerArgs
18
-from cacheflow.server.async_llm_server import AsyncLLMEngine
19
-from cacheflow.server.tokenizer_utils import get_tokenizer
16
+from cacheflow.engine.arg_utils import AsyncEngineArgs
17
+from cacheflow.engine.async_llm_engine import AsyncLLMEngine
18
+from cacheflow.engine.tokenizer_utils import get_tokenizer
19
+from cacheflow.entrypoints.openai.protocol import (
20
+ CompletionRequest, CompletionResponse, CompletionResponseChoice,
21
+ CompletionResponseStreamChoice, CompletionStreamResponse, ErrorResponse,
22
+ LogProbs, ModelCard, ModelList, ModelPermission, UsageInfo)
23
from cacheflow.logger import init_logger
24
+from cacheflow.outputs import RequestOutput
25
from cacheflow.sampling_params import SamplingParams
26
from cacheflow.utils import random_uuid
23
-from cacheflow.entrypoints.openai.protocol import (
24
- CompletionRequest,
25
- CompletionResponse,
26
- CompletionResponseChoice,
27
- CompletionResponseStreamChoice,
28
- CompletionStreamResponse,
29
- ErrorResponse,
30
- LogProbs,
31
- ModelCard,
32
- ModelList,
33
- ModelPermission,
34
- UsageInfo,
35
-)
27
28
TIMEOUT_KEEP_ALIVE = 5 # seconds
29
@@ -102,11 +93,11 @@ async def create_completion(raw_request: Request):
93
for the API specification. This API mimics the OpenAI Completion API.
94
95
NOTE: Currently we do not support the following features:
105
- - echo (since the cacheflow server does not currently support
96
+ - echo (since the cacheflow engine does not currently support
97
getting the logprobs of prompt tokens)
98
- suffix (the language models we currently support do not support
99
suffix)
109
- - logit_bias (to be supported in cacheflow server)
100
+ - logit_bias (to be supported in cacheflow engine)
101
"""
102
request = CompletionRequest(**await raw_request.json())
103
logger.info(f"Received completion request: {request}")
@@ -116,7 +107,7 @@ async def create_completion(raw_request: Request):
107
return error_check_ret
108
109
if request.echo:
119
- # We do not support echo since the cacheflow server does not
110
+ # We do not support echo since the cacheflow engine does not
111
# currently support getting the logprobs of prompt tokens.
112
return create_error_response(HTTPStatus.BAD_REQUEST,
113
"echo is not currently supported")
@@ -127,7 +118,7 @@ async def create_completion(raw_request: Request):
118
"suffix is not currently supported")
119
120
if request.logit_bias is not None:
130
- # TODO: support logit_bias in cacheflow server.
121
+ # TODO: support logit_bias in cacheflow engine.
122
return create_error_response(HTTPStatus.BAD_REQUEST,
123
"logit_bias is not currently supported")
124
@@ -153,7 +144,7 @@ async def create_completion(raw_request: Request):
144
except ValueError as e:
145
return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
146
156
- result_generator = server.generate(prompt, sampling_params,
147
+ result_generator = engine.generate(prompt, sampling_params,
148
request_id)
149
150
# Similar to the OpenAI API, when n != best_of, we do not stream the
@@ -163,7 +154,7 @@ async def create_completion(raw_request: Request):
154
not request.use_beam_search)
155
156
async def abort_request() -> None:
166
- await server.abort(request_id)
157
+ await engine.abort(request_id)
158
159
def create_stream_response_json(index: int,
160
text: str,
@@ -303,7 +294,7 @@ if __name__ == "__main__":
294
help="The model name used in the API. If not specified, "
295
"the model name will be the same as the "
296
"huggingface name.")
306
- parser = AsyncServerArgs.add_cli_args(parser)
297
+ parser = AsyncEngineArgs.add_cli_args(parser)
298
args = parser.parse_args()
299
300
app.add_middleware(
@@ -318,8 +309,8 @@ if __name__ == "__main__":
309
310
served_model = args.served_model_name or args.model
311
321
- server_args = AsyncServerArgs.from_cli_args(args)
322
- server = AsyncLLMEngine.from_server_args(server_args)
312
+ engine_args = AsyncEngineArgs.from_cli_args(args)
313
+ engine = AsyncLLMEngine.from_engine_args(engine_args)
314
315
# A separate tokenizer to map token IDs to strings.
316
tokenizer = get_tokenizer(args.model)
examples/llm_engine_example.py
renamed
+10
-10
@@ -1,12 +1,12 @@
1
import argparse
2
3
-from cacheflow import ServerArgs, LLMEngine, SamplingParams
3
+from cacheflow import EngineArgs, LLMEngine, SamplingParams
4
5
6
def main(args: argparse.Namespace):
7
- # Parse the CLI argument and initialize the server.
8
- server_args = ServerArgs.from_cli_args(args)
9
- server = LLMEngine.from_server_args(server_args)
7
+ # Parse the CLI argument and initialize the engine.
8
+ engine_args = EngineArgs.from_cli_args(args)
9
+ engine = LLMEngine.from_engine_args(engine_args)
10
11
# Test the following prompts.
12
test_prompts = [
@@ -19,27 +19,27 @@ def main(args: argparse.Namespace):
19
SamplingParams(n=3, best_of=3, use_beam_search=True, temperature=0.0)),
20
]
21
22
- # Run the server by calling `server.step()` manually.
22
+ # Run the engine by calling `engine.step()` manually.
23
request_id = 0
24
while True:
25
# To test iteration-level scheduling, we add one request at each step.
26
if test_prompts:
27
prompt, sampling_params = test_prompts.pop(0)
28
- server.add_request(str(request_id), prompt, sampling_params)
28
+ engine.add_request(str(request_id), prompt, sampling_params)
29
request_id += 1
30
31
- request_outputs = server.step()
31
+ request_outputs = engine.step()
32
for request_output in request_outputs:
33
if request_output.finished():
34
print(request_output)
35
36
- if not (server.has_unfinished_requests() or test_prompts):
36
+ if not (engine.has_unfinished_requests() or test_prompts):
37
break
38
39
40
if __name__ == '__main__':
41
parser = argparse.ArgumentParser(
42
- description='Demo on using the LLMEngine class synchronously')
43
- parser = ServerArgs.add_cli_args(parser)
42
+ description='Demo on using the LLMEngine class directly')
43
+ parser = EngineArgs.add_cli_args(parser)
44
args = parser.parse_args()
45
main(args)