A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add docstrings for LLMServer and related classes and examples (#142)
Zhuohan Li committed
Jun 7, 2023 at 18:25 UTC
4298374265a4379a2bd378373c7252b7a7b2b34f
10 files changed
+216
-22
cacheflow/config.py
+39
-3
@@ -12,6 +12,20 @@ _GiB = 1 << 30
12
13
14
class ModelConfig:
15
+ """Configuration for the model.
16
+
17
+ Args:
18
+ model: Name or path of the huggingface model to use.
19
+ download_dir: Directory to download and load the weights, default to the
20
+ default cache directory of huggingface.
21
+ use_np_weights: Save a numpy copy of model weights for faster loading.
22
+ This can increase the disk usage by up to 2x.
23
+ use_dummy_weights: Use dummy values for model weights (for profiling).
24
+ dtype: Data type for model weights and activations. The "auto" option
25
+ will use FP16 precision for FP32 and FP16 models, and BF16 precision
26
+ for BF16 models.
27
+ seed: Random seed for reproducibility.
28
+ """
29
30
def __init__(
31
self,
@@ -68,7 +82,14 @@ class ModelConfig:
82
83
84
class CacheConfig:
71
-
85
+ """Configuration for the KV cache.
86
+
87
+ Args:
88
+ block_size: Size of a cache block in number of tokens.
89
+ gpu_memory_utilization: Fraction of GPU memory to use for the
90
+ CacheFlow execution.
91
+ swap_space: Size of the CPU swap space per GPU (in GiB).
92
+ """
93
def __init__(
94
self,
95
block_size: int,
@@ -111,7 +132,15 @@ class CacheConfig:
132
133
134
class ParallelConfig:
114
-
135
+ """Configuration for the distributed execution.
136
+
137
+ Args:
138
+ pipeline_parallel_size: Number of pipeline parallel groups.
139
+ tensor_parallel_size: Number of tensor parallel groups.
140
+ worker_use_ray: Whether to use Ray for model workers. Will be set to
141
+ True if either pipeline_parallel_size or tensor_parallel_size is
142
+ greater than 1.
143
+ """
144
def __init__(
145
self,
146
pipeline_parallel_size: int,
@@ -134,7 +163,14 @@ class ParallelConfig:
163
164
165
class SchedulerConfig:
137
-
166
+ """Scheduler configuration.
167
+
168
+ Args:
169
+ max_num_batched_tokens: Maximum number of tokens to be processed in
170
+ a single iteration.
171
+ max_num_seqs: Maximum number of sequences to be processed in a single
172
+ iteration.
173
+ """
174
def __init__(
175
self,
176
max_num_batched_tokens: int,
cacheflow/entrypoints/openai/openai_frontend.py
+12
@@ -96,6 +96,18 @@ def create_logprobs(token_ids: List[int],
96
97
@app.post("/v1/completions")
98
async def create_completion(raw_request: Request):
99
+ """Completion API similar to OpenAI's API.
100
+
101
+ See https://platform.openai.com/docs/api-reference/completions/create
102
+ for the API specification. This API mimics the OpenAI Completion API.
103
+
104
+ NOTE: Currently we do not support the following features:
105
+ - echo (since the cacheflow server does not currently support
106
+ getting the logprobs of prompt tokens)
107
+ - suffix (the language models we currently support do not support
108
+ suffix)
109
+ - logit_bias (to be supported in cacheflow server)
110
+ """
111
request = CompletionRequest(**await raw_request.json())
112
logger.info(f"Received completion request: {request}")
113
cacheflow/entrypoints/simple_fastapi_frontend.py
+6
@@ -18,6 +18,12 @@ app = FastAPI()
18
19
@app.post("/generate")
20
async def generate_stream(request: Request) -> StreamingResponse:
21
+ """ Stream the results of the generation request.
22
+
23
+ The request should be a JSON object with the following fields:
24
+ - prompt: the prompt to use for the generation.
25
+ - other fields: the sampling parameters (See `SamplingParams` for details).
26
+ """
27
request_dict = await request.json()
28
prompt = request_dict.pop("prompt")
29
sampling_params = SamplingParams(**request_dict)
cacheflow/server/arg_utils.py
+2
@@ -9,6 +9,7 @@ from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
9
10
@dataclass
11
class ServerArgs:
12
+ """Arguments for CacheFlow servers."""
13
model: str
14
download_dir: Optional[str] = None
15
use_np_weights: bool = False
@@ -117,6 +118,7 @@ class ServerArgs:
118
119
@dataclass
120
class AsyncServerArgs(ServerArgs):
121
+ """Arguments for asynchronous CacheFlow servers."""
122
server_use_ray: bool = False
123
124
@staticmethod
cacheflow/server/async_llm_server.py
+67
-7
@@ -1,6 +1,6 @@
1
import asyncio
2
import time
3
-from typing import Dict, Optional
3
+from typing import Dict, List, Optional
4
5
from cacheflow.logger import init_logger
6
from cacheflow.outputs import RequestOutput
@@ -15,7 +15,25 @@ TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
15
16
17
class AsyncLLMServer:
18
-
18
+ """An asynchronous wrapper for LLMServer.
19
+
20
+ This class is used to wrap the LLMServer class to make it asynchronous. It
21
+ uses asyncio to create a background loop that keeps processing incoming
22
+ requests. The LLMServer is kicked by the generate method when there
23
+ are requests in the waiting queue. The generate method yields the outputs
24
+ from the LLMServer to the caller.
25
+
26
+ NOTE: For the comprehensive list of arguments, see `LLMServer`.
27
+
28
+ Args:
29
+ worker_use_ray: Whether to use Ray for model workers. Required for
30
+ distributed execution. Should be the same as
31
+ `parallel_config.worker_use_ray`.
32
+ server_use_ray: Whether to make LLMServer a Ray actor. If so, the
33
+ async frontend will be executed in a separate process as the
34
+ model workers.
35
+ *args, *kwargs: Arguments for LLMServer.
36
+ """
37
def __init__(self, worker_use_ray: bool, server_use_ray: bool,
38
*args, **kwargs) -> None:
39
self.worker_use_ray = worker_use_ray
@@ -35,6 +53,7 @@ class AsyncLLMServer:
53
self.kicking_request_id: Optional[str] = None
54
55
async def server_step(self, kicking_request_id: Optional[str] = None):
56
+ """Kick the server to process the waiting requests."""
57
self.is_server_running = True
58
self.kicking_request_id = kicking_request_id
59
if self.server_use_ray:
@@ -54,8 +73,31 @@ class AsyncLLMServer:
73
self.request_outputs[request_id] = request_output
74
self.request_events[request_id].set()
75
57
- async def generate(self, prompt: str, sampling_params: SamplingParams,
58
- request_id: str) -> RequestOutput:
76
+ async def generate(
77
+ self,
78
+ prompt: Optional[str],
79
+ sampling_params: SamplingParams,
80
+ request_id: str,
81
+ prompt_token_ids: Optional[List[int]] = None
82
+ ) -> RequestOutput:
83
+ """Generate outputs for a request.
84
+
85
+ Generate outputs for a request. This method is a coroutine. It adds the
86
+ request into the waiting queue of the LLMServer and streams the outputs
87
+ from the LLMServer to the caller.
88
+
89
+ Args:
90
+ prompt: The prompt string. Can be None if prompt_token_ids is
91
+ provided.
92
+ sampling_params: The sampling parameters of the request.
93
+ request_id: The unique id of the request.
94
+ prompt_token_ids: The token IDs of the prompt. If None, we
95
+ use the tokenizer to convert the prompts to token IDs.
96
+
97
+ Yields:
98
+ The output `RequestOutput` objects from the LLMServer for the
99
+ request.
100
+ """
101
# Preprocess the request.
102
arrival_time = time.time()
103
@@ -66,20 +108,29 @@ class AsyncLLMServer:
108
109
logger.info(f"Received request {request_id}: "
110
f"prompt: {prompt!r}, "
69
- f"sampling params: {sampling_params}.")
111
+ f"sampling params: {sampling_params}, "
112
+ f"prompt token ids: {prompt_token_ids}.")
113
114
# Add the request into the cacheflow server's waiting queue.
115
if self.server_use_ray:
116
await self.server.add_request.remote(
74
- request_id, prompt, sampling_params, arrival_time=arrival_time)
117
+ request_id, prompt, sampling_params,
118
+ prompt_token_ids=prompt_token_ids,
119
+ arrival_time=arrival_time)
120
else:
121
self.server.add_request(
77
- request_id, prompt, sampling_params, arrival_time=arrival_time)
122
+ request_id, prompt, sampling_params,
123
+ prompt_token_ids=prompt_token_ids,
124
+ arrival_time=arrival_time)
125
126
# The cacheflow server does not have a background loop that keeps
127
# processing incoming requests. Therefore, we need to keep kicking
128
# the server to process the requests.
129
while True:
130
+ if request_id not in self.request_events:
131
+ # The request has been aborted.
132
+ return
133
+
134
# Kick the server if the server is not running.
135
if not self.is_server_running:
136
await self.server_step(request_id)
@@ -113,6 +164,14 @@ class AsyncLLMServer:
164
break
165
166
async def abort(self, request_id: str) -> None:
167
+ """Abort a request.
168
+
169
+ Abort a submitted request. If the request is finished or not found,
170
+ this method will be a no-op.
171
+
172
+ Args:
173
+ request_id: The unique id of the request.
174
+ """
175
if request_id not in self.request_events:
176
# The request has already finished or been aborted.
177
return
@@ -137,6 +196,7 @@ class AsyncLLMServer:
196
197
@classmethod
198
def from_server_args(cls, server_args: AsyncServerArgs) -> "AsyncLLMServer":
199
+ """Creates an async LLM server from the server arguments."""
200
# Create the server configs.
201
server_configs = server_args.create_server_configs()
202
parallel_config = server_configs[2]
cacheflow/server/llm_server.py
+65
-4
@@ -8,7 +8,7 @@ from cacheflow.logger import init_logger
8
from cacheflow.outputs import RequestOutput
9
from cacheflow.sampling_params import SamplingParams
10
from cacheflow.server.arg_utils import ServerArgs
11
-from cacheflow.server.ray_utils import ray, initialize_cluster
11
+from cacheflow.server.ray_utils import DeviceID, initialize_cluster, ray
12
from cacheflow.server.tokenizer_utils import (get_tokenizer,
13
detokenize_incrementally)
14
from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
@@ -19,6 +19,33 @@ logger = init_logger(__name__)
19
20
21
class LLMServer:
22
+ """An LLM server that receives requests and generates texts.
23
+
24
+ This is the main class for the CacheFlow LLM server. It receives requests
25
+ from clients and generates texts from the LLM. It includes a tokenizer, a
26
+ language model (possibly distributed across multiple GPUs), and GPU memory
27
+ space allocated for intermediate states (aka KV cache). This class utilizes
28
+ iteration-level scheduling and efficient memory management to maximize the
29
+ serving throughput.
30
+
31
+ The `LLM` class wraps this class for offline batched inference and the
32
+ `AsyncLLMServer` class wraps this class for online serving.
33
+
34
+ NOTE: The config arguments are derived from the `ServerArgs` class. For the
35
+ comprehensive list of arguments, see `ServerArgs`.
36
+
37
+ Args:
38
+ model_config: The configuration related to the LLM model.
39
+ cache_config: The configuration related to the KV cache memory
40
+ management.
41
+ parallel_config: The configuration related to distributed execution.
42
+ scheduler_config: The configuration related to the request scheduler.
43
+ distributed_init_method: The initialization method for distributed
44
+ execution. See `torch.distributed.init_process_group` for details.
45
+ stage_devices: The list of devices for each stage. Each stage is a list
46
+ of (rank, node_resource, device) tuples.
47
+ log_stats: Whether to log statistics.
48
+ """
49
50
def __init__(
51
self,
@@ -27,7 +54,7 @@ class LLMServer:
54
parallel_config: ParallelConfig,
55
scheduler_config: SchedulerConfig,
56
distributed_init_method: str,
30
- stage_devices: List[List[Any]],
57
+ stage_devices: List[List[DeviceID]],
58
log_stats: bool,
59
) -> None:
60
logger.info(
@@ -83,6 +110,7 @@ class LLMServer:
110
self.cache_config.verify_with_parallel_config(self.parallel_config)
111
112
def _init_cache(self) -> None:
113
+ """Profiles the memory usage and initializes the KV cache."""
114
# Get the maximum number of blocks that can be allocated on GPU and CPU.
115
num_blocks = self._run_workers(
116
"profile_num_available_blocks",
@@ -108,6 +136,7 @@ class LLMServer:
136
137
@classmethod
138
def from_server_args(cls, server_args: ServerArgs) -> "LLMServer":
139
+ """Creates an LLM server from the server arguments."""
140
# Create the server configs.
141
server_configs = server_args.create_server_configs()
142
parallel_config = server_configs[2]
@@ -126,6 +155,22 @@ class LLMServer:
155
prompt_token_ids: Optional[List[int]] = None,
156
arrival_time: Optional[float] = None,
157
) -> None:
158
+ """Add a request to the server's request pool.
159
+
160
+ The request is added to the request pool and will be processed by the
161
+ scheduler as `server.step()` is called. The exact scheduling policy is
162
+ determined by the scheduler.
163
+
164
+ Args:
165
+ request_id: The unique ID of the request.
166
+ prompt: The prompt string. Can be None if prompt_token_ids is
167
+ provided.
168
+ sampling_params: The sampling parameters for text generation.
169
+ prompt_token_ids: The token IDs of the prompt. If None, we
170
+ use the tokenizer to convert the prompts to token IDs.
171
+ arrival_time: The arrival time of the request. If None, we use
172
+ the current time.
173
+ """
174
if arrival_time is None:
175
arrival_time = time.time()
176
if prompt_token_ids is None:
@@ -148,15 +193,30 @@ class LLMServer:
193
self.scheduler.add_seq_group(seq_group)
194
195
def abort_request(self, request_id: str) -> None:
196
+ """Aborts a request with the given ID.
197
+
198
+ Args:
199
+ request_id: The ID of the request to abort.
200
+ """
201
self.scheduler.abort_seq_group(request_id)
202
203
def get_num_unfinished_requests(self) -> int:
204
+ """Gets the number of unfinished requests."""
205
return self.scheduler.get_num_unfinished_seq_groups()
206
207
def has_unfinished_requests(self) -> bool:
208
+ """Returns True if there are unfinished requests."""
209
return self.scheduler.has_unfinished_seqs()
210
211
def step(self) -> List[RequestOutput]:
212
+ """Performs one decoding iteration and returns newly generated results.
213
+
214
+ This function performs one decoding iteration for the server. It first
215
+ schedules the sequences to be executed in the next iteration and the
216
+ token blocks to be swapped in/out/copy. Then, it executes the model
217
+ and updates the scheduler with the model outputs. Finally, it decodes
218
+ the sequences and returns the newly generated results.
219
+ """
220
seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
221
if (not seq_group_metadata_list) and scheduler_outputs.is_empty():
222
# Nothing to do.
@@ -188,7 +248,7 @@ class LLMServer:
248
return request_outputs
249
250
def _decode_sequences(self, seq_groups: List[SequenceGroup]) -> None:
191
- # Decode the sequence outputs.
251
+ """Decodes the sequence outputs."""
252
for seq_group in seq_groups:
253
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
254
new_token, new_output_text = detokenize_incrementally(
@@ -201,7 +261,7 @@ class LLMServer:
261
seq.output_text = new_output_text
262
263
def _stop_sequences(self, seq_groups: List[SequenceGroup]) -> None:
204
- # Stop the sequences.
264
+ """Stop the finished sequences."""
265
for seq_group in seq_groups:
266
sampling_params = seq_group.sampling_params
267
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
@@ -238,6 +298,7 @@ class LLMServer:
298
*args,
299
**kwargs,
300
) -> Any:
301
+ """Runs the given method on all workers."""
302
all_outputs = []
303
for worker in self.workers:
304
executor = getattr(worker, method)
cacheflow/server/ray_utils.py
+17
-2
@@ -14,15 +14,30 @@ DeviceID = Tuple[int, Optional[str], int] # rank, node resource (node IP), devi
14
def initialize_cluster(
15
parallel_config: ParallelConfig,
16
server_use_ray: bool = False,
17
- address: Optional[str] = None,
17
+ ray_server_address: Optional[str] = None,
18
) -> Tuple[str, List[List[DeviceID]]]:
19
+ """Initialize the distributed cluster probably with Ray.
20
+
21
+ Args:
22
+ parallel_config: The configurations for parallel execution.
23
+ server_use_ray: Whether to use Ray for async server.
24
+ ray_server_address: The address of the Ray cluster. If None, uses
25
+ the default Ray cluster address.
26
+
27
+ Returns:
28
+ A tuple of (`distributed_init_method`, `all_stage_devices`). The
29
+ `distributed_init_method` is the address for initializing the
30
+ distributed backend. `all_stage_devices` includes device IDs for
31
+ each worker in each pipeline stage. Each device ID is a tuple of
32
+ (rank, node resource, device id).
33
+ """
34
if parallel_config.worker_use_ray or server_use_ray:
35
if ray is None:
36
raise ImportError(
37
"Ray is not installed. Please install Ray to use distributed "
38
"serving.")
39
# Connect to a ray cluster.
25
- ray.init(address=address)
40
+ ray.init(address=ray_server_address)
41
42
if not parallel_config.worker_use_ray:
43
# Initialize cluster locally.
cacheflow/server/tokenizer_utils.py
+1
@@ -15,6 +15,7 @@ def get_tokenizer(
15
*args,
16
**kwargs,
17
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
18
+ """Gets a tokenizer for the given model name via Huggingface."""
19
config = AutoConfig.from_pretrained(model_name)
20
if config.model_type == "llama" and getattr(kwargs, "use_fast", True):
21
# LLaMA fast tokenizer causes protobuf errors in some environments.
examples/openai_client.py
+6
-5
@@ -1,14 +1,15 @@
1
import openai
2
+
3
+# Modify OpenAI's API key and API base to use CacheFlow's API server.
4
openai.api_key = "EMPTY"
5
openai.api_base = "http://localhost:8000/v1"
6
model = "facebook/opt-125m"
7
6
-# list models
8
+# Test list models API
9
models = openai.Model.list()
8
-print(models)
9
-
10
-# create a completion
10
+print("Models:", models)
11
12
+# Test completion API
13
stream = True
14
completion = openai.Completion.create(
15
model=model, prompt="A robot may not injure a human being", echo=False, n=2,
@@ -19,4 +20,4 @@ if stream:
20
for c in completion:
21
print(c)
22
else:
22
- print("completion:", completion)
23
+ print("Completion result:", completion)
examples/simple_server.py
+1
-1
@@ -19,7 +19,7 @@ def main(args: argparse.Namespace):
19
SamplingParams(n=3, best_of=3, use_beam_search=True, temperature=0.0)),
20
]
21
22
- # Run the server.
22
+ # Run the server by calling `server.step()` manually.
23
request_id = 0
24
while True:
25
# To test iteration-level scheduling, we add one request at each step.