@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4298374265
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add docstrings for LLMServer and related classes and examples (#142)

Zhuohan Li committed Jun 7, 2023 at 18:25 UTC 4298374265a4379a2bd378373c7252b7a7b2b34f
10 files changed +216 -22
cacheflow/config.py
+39 -3
@@ -12,6 +12,20 @@ _GiB = 1 << 30
12
13
14 class ModelConfig:
15 + """Configuration for the model.
16 +
17 + Args:
18 + model: Name or path of the huggingface model to use.
19 + download_dir: Directory to download and load the weights, default to the
20 + default cache directory of huggingface.
21 + use_np_weights: Save a numpy copy of model weights for faster loading.
22 + This can increase the disk usage by up to 2x.
23 + use_dummy_weights: Use dummy values for model weights (for profiling).
24 + dtype: Data type for model weights and activations. The "auto" option
25 + will use FP16 precision for FP32 and FP16 models, and BF16 precision
26 + for BF16 models.
27 + seed: Random seed for reproducibility.
28 + """
29
30 def __init__(
31 self,
@@ -68,7 +82,14 @@ class ModelConfig:
82
83
84 class CacheConfig:
71 -
85 + """Configuration for the KV cache.
86 +
87 + Args:
88 + block_size: Size of a cache block in number of tokens.
89 + gpu_memory_utilization: Fraction of GPU memory to use for the
90 + CacheFlow execution.
91 + swap_space: Size of the CPU swap space per GPU (in GiB).
92 + """
93 def __init__(
94 self,
95 block_size: int,
@@ -111,7 +132,15 @@ class CacheConfig:
132
133
134 class ParallelConfig:
114 -
135 + """Configuration for the distributed execution.
136 +
137 + Args:
138 + pipeline_parallel_size: Number of pipeline parallel groups.
139 + tensor_parallel_size: Number of tensor parallel groups.
140 + worker_use_ray: Whether to use Ray for model workers. Will be set to
141 + True if either pipeline_parallel_size or tensor_parallel_size is
142 + greater than 1.
143 + """
144 def __init__(
145 self,
146 pipeline_parallel_size: int,
@@ -134,7 +163,14 @@ class ParallelConfig:
163
164
165 class SchedulerConfig:
137 -
166 + """Scheduler configuration.
167 +
168 + Args:
169 + max_num_batched_tokens: Maximum number of tokens to be processed in
170 + a single iteration.
171 + max_num_seqs: Maximum number of sequences to be processed in a single
172 + iteration.
173 + """
174 def __init__(
175 self,
176 max_num_batched_tokens: int,
cacheflow/entrypoints/openai/openai_frontend.py
+12
@@ -96,6 +96,18 @@ def create_logprobs(token_ids: List[int],
96
97 @app.post("/v1/completions")
98 async def create_completion(raw_request: Request):
99 + """Completion API similar to OpenAI's API.
100 +
101 + See https://platform.openai.com/docs/api-reference/completions/create
102 + for the API specification. This API mimics the OpenAI Completion API.
103 +
104 + NOTE: Currently we do not support the following features:
105 + - echo (since the cacheflow server does not currently support
106 + getting the logprobs of prompt tokens)
107 + - suffix (the language models we currently support do not support
108 + suffix)
109 + - logit_bias (to be supported in cacheflow server)
110 + """
111 request = CompletionRequest(**await raw_request.json())
112 logger.info(f"Received completion request: {request}")
113
cacheflow/entrypoints/simple_fastapi_frontend.py
+6
@@ -18,6 +18,12 @@ app = FastAPI()
18
19 @app.post("/generate")
20 async def generate_stream(request: Request) -> StreamingResponse:
21 + """ Stream the results of the generation request.
22 +
23 + The request should be a JSON object with the following fields:
24 + - prompt: the prompt to use for the generation.
25 + - other fields: the sampling parameters (See `SamplingParams` for details).
26 + """
27 request_dict = await request.json()
28 prompt = request_dict.pop("prompt")
29 sampling_params = SamplingParams(**request_dict)
cacheflow/server/arg_utils.py
+2
@@ -9,6 +9,7 @@ from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
9
10 @dataclass
11 class ServerArgs:
12 + """Arguments for CacheFlow servers."""
13 model: str
14 download_dir: Optional[str] = None
15 use_np_weights: bool = False
@@ -117,6 +118,7 @@ class ServerArgs:
118
119 @dataclass
120 class AsyncServerArgs(ServerArgs):
121 + """Arguments for asynchronous CacheFlow servers."""
122 server_use_ray: bool = False
123
124 @staticmethod
cacheflow/server/async_llm_server.py
+67 -7
@@ -1,6 +1,6 @@
1 import asyncio
2 import time
3 -from typing import Dict, Optional
3 +from typing import Dict, List, Optional
4
5 from cacheflow.logger import init_logger
6 from cacheflow.outputs import RequestOutput
@@ -15,7 +15,25 @@ TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
15
16
17 class AsyncLLMServer:
18 -
18 + """An asynchronous wrapper for LLMServer.
19 +
20 + This class is used to wrap the LLMServer class to make it asynchronous. It
21 + uses asyncio to create a background loop that keeps processing incoming
22 + requests. The LLMServer is kicked by the generate method when there
23 + are requests in the waiting queue. The generate method yields the outputs
24 + from the LLMServer to the caller.
25 +
26 + NOTE: For the comprehensive list of arguments, see `LLMServer`.
27 +
28 + Args:
29 + worker_use_ray: Whether to use Ray for model workers. Required for
30 + distributed execution. Should be the same as
31 + `parallel_config.worker_use_ray`.
32 + server_use_ray: Whether to make LLMServer a Ray actor. If so, the
33 + async frontend will be executed in a separate process as the
34 + model workers.
35 + *args, *kwargs: Arguments for LLMServer.
36 + """
37 def __init__(self, worker_use_ray: bool, server_use_ray: bool,
38 *args, **kwargs) -> None:
39 self.worker_use_ray = worker_use_ray
@@ -35,6 +53,7 @@ class AsyncLLMServer:
53 self.kicking_request_id: Optional[str] = None
54
55 async def server_step(self, kicking_request_id: Optional[str] = None):
56 + """Kick the server to process the waiting requests."""
57 self.is_server_running = True
58 self.kicking_request_id = kicking_request_id
59 if self.server_use_ray:
@@ -54,8 +73,31 @@ class AsyncLLMServer:
73 self.request_outputs[request_id] = request_output
74 self.request_events[request_id].set()
75
57 - async def generate(self, prompt: str, sampling_params: SamplingParams,
58 - request_id: str) -> RequestOutput:
76 + async def generate(
77 + self,
78 + prompt: Optional[str],
79 + sampling_params: SamplingParams,
80 + request_id: str,
81 + prompt_token_ids: Optional[List[int]] = None
82 + ) -> RequestOutput:
83 + """Generate outputs for a request.
84 +
85 + Generate outputs for a request. This method is a coroutine. It adds the
86 + request into the waiting queue of the LLMServer and streams the outputs
87 + from the LLMServer to the caller.
88 +
89 + Args:
90 + prompt: The prompt string. Can be None if prompt_token_ids is
91 + provided.
92 + sampling_params: The sampling parameters of the request.
93 + request_id: The unique id of the request.
94 + prompt_token_ids: The token IDs of the prompt. If None, we
95 + use the tokenizer to convert the prompts to token IDs.
96 +
97 + Yields:
98 + The output `RequestOutput` objects from the LLMServer for the
99 + request.
100 + """
101 # Preprocess the request.
102 arrival_time = time.time()
103
@@ -66,20 +108,29 @@ class AsyncLLMServer:
108
109 logger.info(f"Received request {request_id}: "
110 f"prompt: {prompt!r}, "
69 - f"sampling params: {sampling_params}.")
111 + f"sampling params: {sampling_params}, "
112 + f"prompt token ids: {prompt_token_ids}.")
113
114 # Add the request into the cacheflow server's waiting queue.
115 if self.server_use_ray:
116 await self.server.add_request.remote(
74 - request_id, prompt, sampling_params, arrival_time=arrival_time)
117 + request_id, prompt, sampling_params,
118 + prompt_token_ids=prompt_token_ids,
119 + arrival_time=arrival_time)
120 else:
121 self.server.add_request(
77 - request_id, prompt, sampling_params, arrival_time=arrival_time)
122 + request_id, prompt, sampling_params,
123 + prompt_token_ids=prompt_token_ids,
124 + arrival_time=arrival_time)
125
126 # The cacheflow server does not have a background loop that keeps
127 # processing incoming requests. Therefore, we need to keep kicking
128 # the server to process the requests.
129 while True:
130 + if request_id not in self.request_events:
131 + # The request has been aborted.
132 + return
133 +
134 # Kick the server if the server is not running.
135 if not self.is_server_running:
136 await self.server_step(request_id)
@@ -113,6 +164,14 @@ class AsyncLLMServer:
164 break
165
166 async def abort(self, request_id: str) -> None:
167 + """Abort a request.
168 +
169 + Abort a submitted request. If the request is finished or not found,
170 + this method will be a no-op.
171 +
172 + Args:
173 + request_id: The unique id of the request.
174 + """
175 if request_id not in self.request_events:
176 # The request has already finished or been aborted.
177 return
@@ -137,6 +196,7 @@ class AsyncLLMServer:
196
197 @classmethod
198 def from_server_args(cls, server_args: AsyncServerArgs) -> "AsyncLLMServer":
199 + """Creates an async LLM server from the server arguments."""
200 # Create the server configs.
201 server_configs = server_args.create_server_configs()
202 parallel_config = server_configs[2]
cacheflow/server/llm_server.py
+65 -4
@@ -8,7 +8,7 @@ from cacheflow.logger import init_logger
8 from cacheflow.outputs import RequestOutput
9 from cacheflow.sampling_params import SamplingParams
10 from cacheflow.server.arg_utils import ServerArgs
11 -from cacheflow.server.ray_utils import ray, initialize_cluster
11 +from cacheflow.server.ray_utils import DeviceID, initialize_cluster, ray
12 from cacheflow.server.tokenizer_utils import (get_tokenizer,
13 detokenize_incrementally)
14 from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
@@ -19,6 +19,33 @@ logger = init_logger(__name__)
19
20
21 class LLMServer:
22 + """An LLM server that receives requests and generates texts.
23 +
24 + This is the main class for the CacheFlow LLM server. It receives requests
25 + from clients and generates texts from the LLM. It includes a tokenizer, a
26 + language model (possibly distributed across multiple GPUs), and GPU memory
27 + space allocated for intermediate states (aka KV cache). This class utilizes
28 + iteration-level scheduling and efficient memory management to maximize the
29 + serving throughput.
30 +
31 + The `LLM` class wraps this class for offline batched inference and the
32 + `AsyncLLMServer` class wraps this class for online serving.
33 +
34 + NOTE: The config arguments are derived from the `ServerArgs` class. For the
35 + comprehensive list of arguments, see `ServerArgs`.
36 +
37 + Args:
38 + model_config: The configuration related to the LLM model.
39 + cache_config: The configuration related to the KV cache memory
40 + management.
41 + parallel_config: The configuration related to distributed execution.
42 + scheduler_config: The configuration related to the request scheduler.
43 + distributed_init_method: The initialization method for distributed
44 + execution. See `torch.distributed.init_process_group` for details.
45 + stage_devices: The list of devices for each stage. Each stage is a list
46 + of (rank, node_resource, device) tuples.
47 + log_stats: Whether to log statistics.
48 + """
49
50 def __init__(
51 self,
@@ -27,7 +54,7 @@ class LLMServer:
54 parallel_config: ParallelConfig,
55 scheduler_config: SchedulerConfig,
56 distributed_init_method: str,
30 - stage_devices: List[List[Any]],
57 + stage_devices: List[List[DeviceID]],
58 log_stats: bool,
59 ) -> None:
60 logger.info(
@@ -83,6 +110,7 @@ class LLMServer:
110 self.cache_config.verify_with_parallel_config(self.parallel_config)
111
112 def _init_cache(self) -> None:
113 + """Profiles the memory usage and initializes the KV cache."""
114 # Get the maximum number of blocks that can be allocated on GPU and CPU.
115 num_blocks = self._run_workers(
116 "profile_num_available_blocks",
@@ -108,6 +136,7 @@ class LLMServer:
136
137 @classmethod
138 def from_server_args(cls, server_args: ServerArgs) -> "LLMServer":
139 + """Creates an LLM server from the server arguments."""
140 # Create the server configs.
141 server_configs = server_args.create_server_configs()
142 parallel_config = server_configs[2]
@@ -126,6 +155,22 @@ class LLMServer:
155 prompt_token_ids: Optional[List[int]] = None,
156 arrival_time: Optional[float] = None,
157 ) -> None:
158 + """Add a request to the server's request pool.
159 +
160 + The request is added to the request pool and will be processed by the
161 + scheduler as `server.step()` is called. The exact scheduling policy is
162 + determined by the scheduler.
163 +
164 + Args:
165 + request_id: The unique ID of the request.
166 + prompt: The prompt string. Can be None if prompt_token_ids is
167 + provided.
168 + sampling_params: The sampling parameters for text generation.
169 + prompt_token_ids: The token IDs of the prompt. If None, we
170 + use the tokenizer to convert the prompts to token IDs.
171 + arrival_time: The arrival time of the request. If None, we use
172 + the current time.
173 + """
174 if arrival_time is None:
175 arrival_time = time.time()
176 if prompt_token_ids is None:
@@ -148,15 +193,30 @@ class LLMServer:
193 self.scheduler.add_seq_group(seq_group)
194
195 def abort_request(self, request_id: str) -> None:
196 + """Aborts a request with the given ID.
197 +
198 + Args:
199 + request_id: The ID of the request to abort.
200 + """
201 self.scheduler.abort_seq_group(request_id)
202
203 def get_num_unfinished_requests(self) -> int:
204 + """Gets the number of unfinished requests."""
205 return self.scheduler.get_num_unfinished_seq_groups()
206
207 def has_unfinished_requests(self) -> bool:
208 + """Returns True if there are unfinished requests."""
209 return self.scheduler.has_unfinished_seqs()
210
211 def step(self) -> List[RequestOutput]:
212 + """Performs one decoding iteration and returns newly generated results.
213 +
214 + This function performs one decoding iteration for the server. It first
215 + schedules the sequences to be executed in the next iteration and the
216 + token blocks to be swapped in/out/copy. Then, it executes the model
217 + and updates the scheduler with the model outputs. Finally, it decodes
218 + the sequences and returns the newly generated results.
219 + """
220 seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
221 if (not seq_group_metadata_list) and scheduler_outputs.is_empty():
222 # Nothing to do.
@@ -188,7 +248,7 @@ class LLMServer:
248 return request_outputs
249
250 def _decode_sequences(self, seq_groups: List[SequenceGroup]) -> None:
191 - # Decode the sequence outputs.
251 + """Decodes the sequence outputs."""
252 for seq_group in seq_groups:
253 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
254 new_token, new_output_text = detokenize_incrementally(
@@ -201,7 +261,7 @@ class LLMServer:
261 seq.output_text = new_output_text
262
263 def _stop_sequences(self, seq_groups: List[SequenceGroup]) -> None:
204 - # Stop the sequences.
264 + """Stop the finished sequences."""
265 for seq_group in seq_groups:
266 sampling_params = seq_group.sampling_params
267 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
@@ -238,6 +298,7 @@ class LLMServer:
298 *args,
299 **kwargs,
300 ) -> Any:
301 + """Runs the given method on all workers."""
302 all_outputs = []
303 for worker in self.workers:
304 executor = getattr(worker, method)
cacheflow/server/ray_utils.py
+17 -2
@@ -14,15 +14,30 @@ DeviceID = Tuple[int, Optional[str], int] # rank, node resource (node IP), devi
14 def initialize_cluster(
15 parallel_config: ParallelConfig,
16 server_use_ray: bool = False,
17 - address: Optional[str] = None,
17 + ray_server_address: Optional[str] = None,
18 ) -> Tuple[str, List[List[DeviceID]]]:
19 + """Initialize the distributed cluster probably with Ray.
20 +
21 + Args:
22 + parallel_config: The configurations for parallel execution.
23 + server_use_ray: Whether to use Ray for async server.
24 + ray_server_address: The address of the Ray cluster. If None, uses
25 + the default Ray cluster address.
26 +
27 + Returns:
28 + A tuple of (`distributed_init_method`, `all_stage_devices`). The
29 + `distributed_init_method` is the address for initializing the
30 + distributed backend. `all_stage_devices` includes device IDs for
31 + each worker in each pipeline stage. Each device ID is a tuple of
32 + (rank, node resource, device id).
33 + """
34 if parallel_config.worker_use_ray or server_use_ray:
35 if ray is None:
36 raise ImportError(
37 "Ray is not installed. Please install Ray to use distributed "
38 "serving.")
39 # Connect to a ray cluster.
25 - ray.init(address=address)
40 + ray.init(address=ray_server_address)
41
42 if not parallel_config.worker_use_ray:
43 # Initialize cluster locally.
cacheflow/server/tokenizer_utils.py
+1
@@ -15,6 +15,7 @@ def get_tokenizer(
15 *args,
16 **kwargs,
17 ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
18 + """Gets a tokenizer for the given model name via Huggingface."""
19 config = AutoConfig.from_pretrained(model_name)
20 if config.model_type == "llama" and getattr(kwargs, "use_fast", True):
21 # LLaMA fast tokenizer causes protobuf errors in some environments.
examples/openai_client.py
+6 -5
@@ -1,14 +1,15 @@
1 import openai
2 +
3 +# Modify OpenAI's API key and API base to use CacheFlow's API server.
4 openai.api_key = "EMPTY"
5 openai.api_base = "http://localhost:8000/v1"
6 model = "facebook/opt-125m"
7
6 -# list models
8 +# Test list models API
9 models = openai.Model.list()
8 -print(models)
9 -
10 -# create a completion
10 +print("Models:", models)
11
12 +# Test completion API
13 stream = True
14 completion = openai.Completion.create(
15 model=model, prompt="A robot may not injure a human being", echo=False, n=2,
@@ -19,4 +20,4 @@ if stream:
20 for c in completion:
21 print(c)
22 else:
22 - print("completion:", completion)
23 + print("Completion result:", completion)
examples/simple_server.py
+1 -1
@@ -19,7 +19,7 @@ def main(args: argparse.Namespace):
19 SamplingParams(n=3, best_of=3, use_beam_search=True, temperature=0.0)),
20 ]
21
22 - # Run the server.
22 + # Run the server by calling `server.step()` manually.
23 request_id = 0
24 while True:
25 # To test iteration-level scheduling, we add one request at each step.