@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c3442c1f6f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Refactor system architecture (#109)

Woosuk Kwon committed May 20, 2023 at 13:06 UTC c3442c1f6fabe54adb82d2d676920c5f31b9834e
24 files changed +1019 -1036
README.md
+8 -5
@@ -10,13 +10,17 @@ pip install -e . # This may take several minutes.
10 ## Test simple server
11
12 ```bash
13 +# Single-GPU inference.
14 +python examples/simple_server.py # --model <your_model>
15 +
16 +# Multi-GPU inference (e.g., 2 GPUs).
17 ray start --head
14 -python simple_server.py
18 +python examples/simple_server.py -tp 2 # --model <your_model>
19 ```
20
21 The detailed arguments for `simple_server.py` can be found by:
22 ```bash
19 -python simple_server.py --help
23 +python examples/simple_server.py --help
24 ```
25
26 ## FastAPI server
@@ -24,12 +28,12 @@ python simple_server.py --help
28 To start the server:
29 ```bash
30 ray start --head
27 -python -m cacheflow.http_frontend.fastapi_frontend
31 +python -m cacheflow.entrypoints.fastapi_server # --model <your_model>
32 ```
33
34 To test the server:
35 ```bash
32 -python -m cacheflow.http_frontend.test_cli_client
36 +python test_cli_client.py
37 ```
38
39 ## Gradio web server
@@ -55,7 +59,6 @@ Since LLaMA weight is not fully public, we cannot directly download the LLaMA we
59 python src/transformers/models/llama/convert_llama_weights_to_hf.py \
60 --input_dir /path/to/downloaded/llama/weights --model_size 7B --output_dir /output/path/llama-7b
61 ```
58 - Please make sure that `llama` is included in the output directory name.
62 2. For all the commands above, specify the model with `--model /output/path/llama-7b` to load the model. For example:
63 ```bash
64 python simple_server.py --model /output/path/llama-7b
cacheflow/__init__.py new
+19
@@ -0,0 +1,19 @@
1 +from cacheflow.outputs import RequestOutput
2 +from cacheflow.sampling_params import SamplingParams
3 +from cacheflow.server.arg_utils import (
4 + add_server_arguments,
5 + create_server_configs_from_args,
6 + initialize_server_from_args,
7 +)
8 +from cacheflow.server.llm_server import LLMServer
9 +from cacheflow.server.ray_utils import initialize_cluster
10 +
11 +__all__ = [
12 + "RequestOutput",
13 + "SamplingParams",
14 + "LLMServer",
15 + "add_server_arguments",
16 + "create_server_configs_from_args",
17 + "initialize_server_from_args",
18 + "initialize_cluster",
19 +]
cacheflow/config.py new
+165
@@ -0,0 +1,165 @@
1 +from typing import Optional
2 +
3 +import torch
4 +from transformers import AutoConfig, PretrainedConfig
5 +
6 +
7 +class ModelConfig:
8 +
9 + def __init__(
10 + self,
11 + model: str,
12 + download_dir: Optional[str],
13 + use_np_weights: bool,
14 + use_dummy_weights: bool,
15 + dtype: str,
16 + seed: int,
17 + ) -> None:
18 + self.model = model
19 + self.download_dir = download_dir
20 + self.use_np_weights = use_np_weights
21 + self.use_dummy_weights = use_dummy_weights
22 + self.seed = seed
23 +
24 + self.hf_config: PretrainedConfig = AutoConfig.from_pretrained(model)
25 + self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
26 +
27 + def verify_with_parallel_config(
28 + self,
29 + parallel_config: "ParallelConfig",
30 + ) -> None:
31 + total_num_attention_heads = self.hf_config.num_attention_heads
32 + tensor_parallel_size = parallel_config.tensor_parallel_size
33 + if total_num_attention_heads % tensor_parallel_size != 0:
34 + raise ValueError(
35 + f"Total number of attention heads ({total_num_attention_heads})"
36 + " must be divisible by tensor parallel size "
37 + f"({tensor_parallel_size}).")
38 +
39 + total_num_hidden_layers = self.hf_config.num_hidden_layers
40 + pipeline_parallel_size = parallel_config.pipeline_parallel_size
41 + if total_num_hidden_layers % pipeline_parallel_size != 0:
42 + raise ValueError(
43 + f"Total number of hidden layers ({total_num_hidden_layers}) "
44 + "must be divisible by pipeline parallel size "
45 + f"({pipeline_parallel_size}).")
46 +
47 + def get_hidden_size(self) -> int:
48 + return self.hf_config.hidden_size
49 +
50 + def get_head_size(self) -> int:
51 + # FIXME(woosuk): This may not be true for all models.
52 + return self.hf_config.hidden_size // self.hf_config.num_attention_heads
53 +
54 + def get_num_heads(self, parallel_config: "ParallelConfig") -> int:
55 + total_num_attention_heads = self.hf_config.num_attention_heads
56 + return total_num_attention_heads // parallel_config.tensor_parallel_size
57 +
58 + def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
59 + total_num_hidden_layers = self.hf_config.num_hidden_layers
60 + return total_num_hidden_layers // parallel_config.pipeline_parallel_size
61 +
62 +
63 +class CacheConfig:
64 +
65 + def __init__(
66 + self,
67 + block_size: int,
68 + gpu_memory_utilization: float,
69 + swap_space: int,
70 + ) -> None:
71 + self.block_size = block_size
72 + self.gpu_memory_utilization = gpu_memory_utilization
73 + self.swap_space = swap_space
74 +
75 + # Will be set after profiling.
76 + self.num_gpu_blocks = None
77 + self.num_cpu_blocks = None
78 +
79 +
80 +class ParallelConfig:
81 +
82 + def __init__(
83 + self,
84 + pipeline_parallel_size: int,
85 + tensor_parallel_size: int,
86 + use_ray: bool,
87 + ) -> None:
88 + self.pipeline_parallel_size = pipeline_parallel_size
89 + self.tensor_parallel_size = tensor_parallel_size
90 + self.use_ray = use_ray
91 +
92 + self.world_size = pipeline_parallel_size * tensor_parallel_size
93 + if self.world_size > 1:
94 + self.use_ray = True
95 + self._verify_args()
96 +
97 + def _verify_args(self) -> None:
98 + if self.pipeline_parallel_size > 1:
99 + raise NotImplementedError(
100 + "Pipeline parallelism is not supported yet.")
101 +
102 +
103 +class SchedulerConfig:
104 +
105 + def __init__(
106 + self,
107 + max_num_batched_tokens: int,
108 + max_num_seqs: int,
109 + ) -> None:
110 + self.max_num_batched_tokens = max_num_batched_tokens
111 + self.max_num_seqs = max_num_seqs
112 +
113 +
114 +_STR_DTYPE_TO_TORCH_DTYPE = {
115 + "half": torch.float16,
116 + "float16": torch.float16,
117 + "float": torch.float32,
118 + "float32": torch.float32,
119 + "bfloat16": torch.bfloat16,
120 +}
121 +
122 +
123 +def _get_and_verify_dtype(
124 + config: PretrainedConfig,
125 + dtype: str,
126 +) -> torch.dtype:
127 + # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
128 + # because config.torch_dtype can be None.
129 + config_dtype = getattr(config, "torch_dtype", None)
130 + if config_dtype is None:
131 + config_dtype = torch.float32
132 +
133 + dtype = dtype.lower()
134 + if dtype == "default":
135 + if config_dtype == torch.float32:
136 + # Following the common practice, we use float16 for float32 models.
137 + torch_dtype = torch.float16
138 + else:
139 + torch_dtype = config_dtype
140 + else:
141 + torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
142 +
143 + # Verify the dtype.
144 + if torch_dtype != config_dtype:
145 + if torch_dtype == torch.float32:
146 + # Upcasting to float32 is allowed.
147 + pass
148 + elif config_dtype == torch.float32:
149 + # Downcasting from float32 to float16 or bfloat16 is allowed.
150 + pass
151 + else:
152 + # Casting between float16 and bfloat16 is not allowed.
153 + raise ValueError(
154 + f"Cannot use {torch_dtype} for {config_dtype} model.")
155 +
156 + # Check if the GPU supports the dtype.
157 + if torch_dtype == torch.bfloat16:
158 + compute_capability = torch.cuda.get_device_capability()
159 + if compute_capability[0] < 8:
160 + gpu_name = torch.cuda.get_device_name()
161 + raise ValueError(
162 + "Bfloat16 is only supported on GPUs with compute capability "
163 + f"of at least 8.0. Your {gpu_name} GPU has compute capability "
164 + f"{compute_capability[0]}.{compute_capability[1]}.")
165 + return torch_dtype
cacheflow/core/scheduler.py
+80 -84
@@ -2,10 +2,10 @@ import enum
2 import time
3 from typing import Dict, List, Optional, Tuple
4
5 +from cacheflow.config import CacheConfig, SchedulerConfig
6 from cacheflow.core.block_manager import BlockSpaceManager
7 from cacheflow.core.policy import PolicyFactory
8 from cacheflow.logger import init_logger
8 -from cacheflow.sampling_params import SamplingParams
9 from cacheflow.sequence import (Sequence, SequenceData, SequenceGroup,
10 SequenceGroupMetadata, SequenceOutputs,
11 SequenceStatus)
@@ -28,43 +28,53 @@ class PreemptionMode(enum.Enum):
28 RECOMPUTE = enum.auto()
29
30
31 +class SchedulerOutputs:
32 +
33 + def __init__(
34 + self,
35 + blocks_to_swap_in: Dict[int, int],
36 + blocks_to_swap_out: Dict[int, int],
37 + blocks_to_copy: Dict[int, List[int]],
38 + ) -> None:
39 + self.blocks_to_swap_in = blocks_to_swap_in
40 + self.blocks_to_swap_out = blocks_to_swap_out
41 + self.blocks_to_copy = blocks_to_copy
42 + # Swap in and swap out should never happen at the same time.
43 + assert not (blocks_to_swap_in and blocks_to_swap_out)
44 +
45 + def is_empty(self) -> bool:
46 + return (not self.blocks_to_swap_in
47 + and not self.blocks_to_swap_out
48 + and not self.blocks_to_copy)
49 +
50 +
51 class Scheduler:
52
53 def __init__(
54 self,
35 - controllers: List,
36 - block_size: int,
37 - num_gpu_blocks: int,
38 - num_cpu_blocks: int,
39 - max_num_batched_tokens: int,
40 - max_num_sequences: int,
55 + scheduler_config: SchedulerConfig,
56 + cache_config: CacheConfig,
57 log_stats: bool,
58 ) -> None:
43 - self.controllers = controllers
44 - self.block_size = block_size
45 - self.num_gpu_blocks = num_gpu_blocks
46 - self.num_cpu_blocks = num_cpu_blocks
47 - self.max_num_batched_tokens = max_num_batched_tokens
48 - self.max_num_sequences = max_num_sequences
59 + self.scheduler_config = scheduler_config
60 + self.cache_config = cache_config
61 self.log_stats = log_stats
62
63 # Instantiate the scheduling policy.
64 self.policy = PolicyFactory.get_policy(policy_name='fcfs')
65 # Create the block space manager.
66 self.block_manager = BlockSpaceManager(
55 - block_size=block_size,
56 - num_gpu_blocks=num_gpu_blocks,
57 - num_cpu_blocks=num_cpu_blocks,
67 + block_size=self.cache_config.block_size,
68 + num_gpu_blocks=self.cache_config.num_gpu_blocks,
69 + num_cpu_blocks=self.cache_config.num_cpu_blocks,
70 )
71
72 # Sequence groups in the WAITING state.
73 self.waiting: List[SequenceGroup] = []
74 # Sequence groups in the RUNNING state.
75 self.running: List[SequenceGroup] = []
64 - # Mapping: group_id -> num_steps.
65 - self.num_steps: Dict[int, int] = {}
66 - # Mapping: group_id -> sampling params.
67 - self.sampling_params: Dict[int, SamplingParams] = {}
76 + # Mapping: request_id -> num_steps.
77 + self.num_steps: Dict[str, int] = {}
78 # Sequence groups in the SWAPPED state.
79 self.swapped: List[SequenceGroup] = []
80
@@ -72,18 +82,15 @@ class Scheduler:
82 # List[timestamp, num_tokens]
83 self.num_input_tokens: List[Tuple[float, int]] = []
84
75 - def add_sequence_groups(
76 - self,
77 - seq_groups: List[Tuple[SequenceGroup, SamplingParams]],
78 - ) -> None:
85 + def add_seq_group(self, seq_group: SequenceGroup) -> None:
86 # Add sequence groups to the waiting queue.
80 - for seq_group, sampling_params in seq_groups:
81 - self.waiting.append(seq_group)
82 - self.sampling_params[seq_group.group_id] = sampling_params
87 + assert seq_group.request_id not in self.num_steps
88 + self.waiting.append(seq_group)
89
84 - def _schedule(
85 - self,
86 - ) -> Tuple[Dict[int, int], Dict[int, int], Dict[int, List[int]], List[int]]:
90 + def has_unfinished_seqs(self) -> bool:
91 + return self.waiting or self.running or self.swapped
92 +
93 + def _schedule(self) -> Tuple[SchedulerOutputs, List[int]]:
94 # Blocks that need to be swaped or copied before model execution.
95 blocks_to_swap_in: Dict[int, int] = {}
96 blocks_to_swap_out: Dict[int, int] = {}
@@ -136,8 +143,9 @@ class Scheduler:
143
144 # The total number of sequences in the RUNNING state should not
145 # exceed the maximum number of sequences.
139 - num_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
140 - if len(self.running) + num_seqs > self.max_num_sequences:
146 + num_new_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
147 + num_curr_seqs = len(self.running)
148 + if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
149 break
150
151 seq_group = self.swapped.pop(0)
@@ -151,7 +159,7 @@ class Scheduler:
159 )
160
161 # Join waiting sequences if possible.
154 - prompt_group_ids: List[int] = []
162 + prompt_group_ids: List[str] = []
163 # NOTE(woosuk): The sequence groups in the SWAPPED state are strictly
164 # prioritized over the sequence groups in the WAITING state.
165 # This is because we want to bound the amount of CPU memory taken by
@@ -172,25 +180,31 @@ class Scheduler:
180 # If the number of batched tokens exceeds the limit, stop.
181 num_prompt_tokens = seq_group.seqs[0].get_len()
182 if (num_batched_tokens + num_prompt_tokens
175 - > self.max_num_batched_tokens):
183 + > self.scheduler_config.max_num_batched_tokens):
184 break
185
186 # The total number of sequences in the RUNNING state should not
187 # exceed the maximum number of sequences.
180 - num_seqs = seq_group.num_seqs(status=SequenceStatus.WAITING)
181 - if len(self.running) + num_seqs > self.max_num_sequences:
188 + num_new_seqs = seq_group.num_seqs(status=SequenceStatus.WAITING)
189 + num_curr_seqs = len(self.running)
190 + if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
191 break
192
193 seq_group = self.waiting.pop(0)
194 self._allocate(seq_group)
195 self.running.append(seq_group)
196 num_batched_tokens += num_prompt_tokens
188 - prompt_group_ids.append(seq_group.group_id)
197 + prompt_group_ids.append(seq_group.request_id)
198
199 + scheduler_outputs = SchedulerOutputs(
200 + blocks_to_swap_in=blocks_to_swap_in,
201 + blocks_to_swap_out=blocks_to_swap_out,
202 + blocks_to_copy=blocks_to_copy,
203 + )
204 if not self.log_stats:
191 - return (blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy,
192 - prompt_group_ids)
205 + return scheduler_outputs, prompt_group_ids
206
207 + # TODO(woosuk): Move the below code to server.
208 now = time.time()
209 if num_batched_tokens > 0:
210 self.num_input_tokens.append((now, num_batched_tokens))
@@ -208,13 +222,16 @@ class Scheduler:
222 else:
223 avg_throughput = 0.0
224
225 + total_num_gpu_blocks = self.cache_config.num_gpu_blocks
226 num_free_gpu_blocks = self.block_manager.get_num_free_gpu_blocks()
212 - num_used_gpu_blocks = self.num_gpu_blocks - num_free_gpu_blocks
213 - gpu_cache_usage = num_used_gpu_blocks / self.num_gpu_blocks
214 - if self.num_cpu_blocks > 0:
227 + num_used_gpu_blocks = total_num_gpu_blocks - num_free_gpu_blocks
228 + gpu_cache_usage = num_used_gpu_blocks / total_num_gpu_blocks
229 +
230 + total_num_cpu_blocks = self.cache_config.num_cpu_blocks
231 + if total_num_cpu_blocks > 0:
232 num_free_cpu_blocks = self.block_manager.get_num_free_cpu_blocks()
216 - num_used_cpu_blocks = self.num_cpu_blocks - num_free_cpu_blocks
217 - cpu_cache_usage = num_used_cpu_blocks / self.num_cpu_blocks
233 + num_used_cpu_blocks = total_num_cpu_blocks - num_free_cpu_blocks
234 + cpu_cache_usage = num_used_cpu_blocks / total_num_cpu_blocks
235 else:
236 cpu_cache_usage = 0.0
237
@@ -225,27 +242,18 @@ class Scheduler:
242 f"Pending: {len(self.waiting)} reqs, "
243 f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
244 f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
245 + return scheduler_outputs, prompt_group_ids
246
229 - return (blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy,
230 - prompt_group_ids)
231 -
232 - def step(self) -> List[SequenceGroup]:
247 + def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs]:
248 # Schedule sequence groups.
249 # This function call changes the internal states of the scheduler
250 # such as self.running, self.swapped, and self.waiting.
236 - scheduler_output = self._schedule()
237 - blocks_to_swap_in = scheduler_output[0]
238 - blocks_to_swap_out = scheduler_output[1]
239 - blocks_to_copy = scheduler_output[2]
240 - prompt_group_ids = scheduler_output[3]
251 + scheduler_outputs, prompt_group_ids = self._schedule()
252
253 # Create input data structures.
254 seq_group_metadata_list: List[SequenceGroupMetadata] = []
244 - updated_seq_groups: List[SequenceGroup] = self.running.copy()
245 -
255 for seq_group in self.running:
247 - group_id = seq_group.group_id
248 - is_prompt = group_id in prompt_group_ids
256 + is_prompt = seq_group.request_id in prompt_group_ids
257
258 seq_data: Dict[int, List[SequenceData]] = {}
259 block_tables: Dict[int, List[int]] = {}
@@ -255,36 +263,24 @@ class Scheduler:
263 block_tables[seq_id] = self.block_manager.get_block_table(seq)
264
265 seq_group_metadata = SequenceGroupMetadata(
258 - group_id=group_id,
266 + request_id=seq_group.request_id,
267 is_prompt=is_prompt,
268 seq_data=seq_data,
261 - sampling_params=self.sampling_params[group_id],
269 + sampling_params=seq_group.sampling_params,
270 block_tables=block_tables,
271 )
272 seq_group_metadata_list.append(seq_group_metadata)
273 + return seq_group_metadata_list, scheduler_outputs
274
266 - # Execute the first stage of the pipeline.
267 - if seq_group_metadata_list or blocks_to_swap_in or blocks_to_swap_out:
268 - # Swap in and swap out should never happen at the same time.
269 - assert not (blocks_to_swap_in and blocks_to_swap_out)
270 - self.controllers[0].execute_stage(
271 - seq_group_metadata_list,
272 - blocks_to_swap_in=blocks_to_swap_in,
273 - blocks_to_swap_out=blocks_to_swap_out,
274 - blocks_to_copy=blocks_to_copy,
275 - )
276 -
277 - return updated_seq_groups
278 -
279 - def post_step(
275 + def update(
276 self,
277 seq_outputs: Dict[int, SequenceOutputs],
282 - ) -> None:
278 + ) -> List[SequenceGroup]:
279 # Update the running sequences and free blocks.
280 for seq_group in self.running:
285 - group_id = seq_group.group_id
286 - self.num_steps[group_id] += 1
287 - stop_token_ids = self.sampling_params[group_id].stop_token_ids
281 + request_id = seq_group.request_id
282 + self.num_steps[request_id] += 1
283 + stop_token_ids = seq_group.sampling_params.stop_token_ids
284
285 # Process beam search results before processing the next tokens.
286 for seq in seq_group.seqs:
@@ -316,12 +312,13 @@ class Scheduler:
312 continue
313
314 # Check if the sequence has reached the maximum number of steps.
319 - max_num_steps = self.sampling_params[group_id].max_tokens
320 - if self.num_steps[group_id] == max_num_steps:
315 + max_num_steps = seq_group.sampling_params.max_tokens
316 + if self.num_steps[request_id] == max_num_steps:
317 self._free_seq(seq)
318 continue
319
320 # Update the running sequences.
321 + updated = self.running.copy()
322 running: List[SequenceGroup] = []
323 for seq_group in self.running:
324 if seq_group.is_finished():
@@ -329,13 +326,14 @@ class Scheduler:
326 else:
327 running.append(seq_group)
328 self.running = running
329 + return updated
330
331 def _allocate(self, seq_group: SequenceGroup) -> None:
332 self.block_manager.allocate(seq_group)
333 for seq in seq_group.seqs:
334 seq.status = SequenceStatus.RUNNING
337 - if seq_group.group_id not in self.num_steps:
338 - self.num_steps[seq_group.group_id] = 0
335 + if seq_group.request_id not in self.num_steps:
336 + self.num_steps[seq_group.request_id] = 0
337
338 def _append_slot(
339 self,
@@ -410,9 +408,7 @@ class Scheduler:
408 self.block_manager.free(seq)
409
410 def _free_seq_group(self, seq_group: SequenceGroup) -> None:
413 - group_id = seq_group.group_id
414 - del self.num_steps[group_id]
415 - del self.sampling_params[group_id]
411 + del self.num_steps[seq_group.request_id]
412
413 def _swap_in(
414 self,
cacheflow/core/server.py deleted
-302
@@ -1,302 +0,0 @@
1 -import argparse
2 -import random
3 -from typing import List, Optional, Tuple
4 -
5 -try:
6 - import ray
7 -except ImportError:
8 - ray = None
9 -import numpy as np
10 -import torch
11 -
12 -from cacheflow.core.scheduler import Scheduler
13 -from cacheflow.frontend.simple_frontend import SimpleFrontend
14 -from cacheflow.logger import init_logger
15 -from cacheflow.sampling_params import SamplingParams
16 -from cacheflow.sequence import SequenceGroup
17 -from cacheflow.worker.controller import Controller, DeviceID
18 -
19 -logger = init_logger(__name__)
20 -
21 -
22 -class Server:
23 -
24 - def __init__(
25 - self,
26 - model: str,
27 - cache_dir: Optional[str],
28 - use_dummy_weights: bool,
29 - use_np_cache: bool,
30 - pipeline_parallel_size: int,
31 - tensor_parallel_size: int,
32 - block_size: int,
33 - dtype: str,
34 - seed: int,
35 - swap_space: int,
36 - gpu_memory_utilization: float,
37 - max_num_batched_tokens: int,
38 - max_num_sequences: int,
39 - num_nodes: int,
40 - num_devices_per_node: int,
41 - distributed_init_method: str,
42 - all_stage_devices: List[List[DeviceID]],
43 - use_ray: bool,
44 - log_stats: bool,
45 - ):
46 - logger.info(
47 - "Initializing a server with config: "
48 - f"model={model!r}, "
49 - f"dtype={dtype}, "
50 - f"use_dummy_weights={use_dummy_weights}, "
51 - f"cache_dir={cache_dir!r}, "
52 - f"use_np_cache={use_np_cache}, "
53 - f"tensor_parallel_size={tensor_parallel_size}, "
54 - f"seed={seed})"
55 - )
56 - self.num_nodes = num_nodes
57 - self.num_devices_per_node = num_devices_per_node
58 - self.world_size = pipeline_parallel_size * tensor_parallel_size
59 -
60 - if not use_ray:
61 - assert self.world_size == 1, (
62 - "Only support single GPU without Ray.")
63 -
64 - # Create a controller for each pipeline stage.
65 - self.controllers: List[Controller] = []
66 - for i in range(pipeline_parallel_size):
67 - controller = Controller(
68 - stage_id=i,
69 - stage_devices=all_stage_devices[i],
70 - world_size=self.world_size,
71 - pipeline_parallel_size=pipeline_parallel_size,
72 - tensor_parallel_size=tensor_parallel_size,
73 - distributed_init_method=distributed_init_method,
74 - model_name=model,
75 - dtype=dtype,
76 - seed=seed,
77 - cache_dir=cache_dir,
78 - use_dummy_weights=use_dummy_weights,
79 - use_np_cache=use_np_cache,
80 - max_num_batched_tokens=max_num_batched_tokens,
81 - max_num_sequences=max_num_sequences,
82 - use_ray=use_ray,
83 - )
84 - self.controllers.append(controller)
85 -
86 - # Initialize cache engine.
87 - all_worker_num_available_blocks = []
88 - for controller in self.controllers:
89 - all_worker_num_available_blocks.extend(
90 - controller.get_num_available_blocks(
91 - block_size, swap_space, gpu_memory_utilization)
92 - )
93 - # Since we use a shared centralized controller, we take the minimum
94 - # number of blocks across all workers to make sure all the memory
95 - # operators can be applied to all workers.
96 - self.num_gpu_blocks = np.min([b[0] for b in all_worker_num_available_blocks])
97 - self.num_cpu_blocks = np.min([b[1] for b in all_worker_num_available_blocks])
98 - logger.info(f'# GPU blocks: {self.num_gpu_blocks}, '
99 - f'# CPU blocks: {self.num_cpu_blocks}')
100 - for controller in self.controllers:
101 - controller.init_cache_engine(block_size, self.num_gpu_blocks,
102 - self.num_cpu_blocks)
103 -
104 - # Create a scheduler.
105 - self.scheduler = Scheduler(
106 - controllers=self.controllers,
107 - block_size=block_size,
108 - num_gpu_blocks=self.num_gpu_blocks,
109 - num_cpu_blocks=self.num_cpu_blocks,
110 - max_num_batched_tokens=max_num_batched_tokens,
111 - max_num_sequences=max_num_sequences,
112 - log_stats=log_stats,
113 - )
114 - # Connect the controllers.
115 - for i in range(len(self.controllers) - 1):
116 - self.controllers[i].set_next(self.controllers[i + 1])
117 - self.controllers[-1].set_next(self.scheduler)
118 -
119 - def add_sequence_groups(
120 - self,
121 - sequence_groups: List[Tuple[SequenceGroup, SamplingParams]]
122 - ):
123 - self.scheduler.add_sequence_groups(sequence_groups)
124 -
125 - def step(self):
126 - return self.scheduler.step()
127 -
128 - def has_unfinished_requests(self):
129 - return (self.scheduler.waiting or self.scheduler.running or
130 - self.scheduler.swapped)
131 -
132 -
133 -def initialize_cluster(
134 - use_ray: bool = False,
135 - address: Optional[str] = None,
136 - pipeline_parallel_size: int = 1,
137 - tensor_parallel_size: int = 1,
138 -) -> Tuple[int, int, str, List[List[DeviceID]]]:
139 - # Initialize cluster locally.
140 - if not use_ray:
141 - assert pipeline_parallel_size * tensor_parallel_size == 1, (
142 - "Only support single GPU without Ray.")
143 - num_nodes = 1
144 - num_devices_per_node = torch.cuda.device_count()
145 - port = random.randint(10000, 20000)
146 - # We need to setup the distributed init method to make sure
147 - # the distributed megatron code (e.g., get world size) works correctly.
148 - distributed_init_method = f"tcp://localhost:{port}"
149 - all_stage_devices = [[(0, None, 0)]]
150 - return (num_nodes, num_devices_per_node, distributed_init_method,
151 - all_stage_devices)
152 -
153 - assert ray is not None, (
154 - "Ray is not installed. Please install Ray to use distributed "
155 - "serving.")
156 -
157 - # Connect to a ray cluster.
158 - ray.init(address=address)
159 -
160 - # Assume we have a uniform cluster that each node has the same number of
161 - # GPUs for now.
162 - valid_node_resources = []
163 - num_devices_per_node = None
164 - for node in ray.nodes():
165 - if (not node['Alive']) or node['Resources']['GPU'] <= 0:
166 - continue
167 - if num_devices_per_node is None:
168 - num_devices_per_node = node['Resources']['GPU']
169 - else:
170 - assert num_devices_per_node == node['Resources']['GPU'], (
171 - "The number of GPUs per node is not uniform.")
172 - for key in node['Resources']:
173 - if key.startswith('node:'):
174 - valid_node_resources.append(key)
175 -
176 - num_nodes = len(valid_node_resources)
177 -
178 - assert (pipeline_parallel_size * tensor_parallel_size
179 - <= num_nodes * num_devices_per_node), (
180 - "The number of required GPUs exceeds the total number of "
181 - "available GPUs.")
182 - if tensor_parallel_size >= num_devices_per_node:
183 - assert tensor_parallel_size % num_devices_per_node == 0, (
184 - "The number of tensor parallelism is not divisible by the "
185 - "number of GPUs per node.")
186 - else:
187 - assert num_devices_per_node % tensor_parallel_size == 0, (
188 - "The number of GPUs per node is not divisible by the number "
189 - "of tensor parallelism.")
190 -
191 - # Assign GPUs to pipeline stages.
192 - rank = 0
193 - current_node_id = 0
194 - current_device_id = 0
195 - distributed_init_method = None
196 - all_stage_devices = []
197 -
198 - for i in range(pipeline_parallel_size):
199 - stage_devices = []
200 - for j in range(tensor_parallel_size):
201 - node_resource = valid_node_resources[current_node_id]
202 - stage_devices.append((rank, node_resource, current_device_id))
203 - if distributed_init_method is None:
204 - ip = node_resource.split("node:")[-1]
205 - port = random.randint(10000, 20000)
206 - distributed_init_method = f"tcp://{ip}:{port}"
207 - rank += 1
208 - current_device_id += 1
209 - if current_device_id >= num_devices_per_node:
210 - current_node_id += 1
211 - current_device_id = 0
212 - all_stage_devices.append(stage_devices)
213 -
214 - return (num_nodes, num_devices_per_node, distributed_init_method,
215 - all_stage_devices)
216 -
217 -
218 -_GiB = 1 << 30
219 -
220 -
221 -def add_server_arguments(parser: argparse.ArgumentParser):
222 - """Shared arguments for CacheFlow servers."""
223 - # Model arguments
224 - parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
225 - parser.add_argument('--cache-dir', type=str, default=None,
226 - help='cache dir to download and load the weights, '
227 - 'default to the default cache dir of huggingface')
228 - parser.add_argument('--use-np-cache', action='store_true',
229 - help='save a numpy copy of model weights for faster loading')
230 - parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
231 - # TODO(woosuk): Support FP32 for debugging.
232 - parser.add_argument('--dtype', type=str, default='default', choices=['default', 'half', 'bfloat16'],
233 - help=('data type for model weights and activations. '
234 - 'The "default" option will use FP16 precision '
235 - 'for FP32 and FP16 models, and BF16 precision '
236 - 'for BF16 models.'))
237 - # Parallel arguments
238 - parser.add_argument('--use-ray', action='store_true', help='use Ray for distributed serving, will be automatically set when using more than 1 GPU')
239 - parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
240 - parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
241 - # KV cache arguments
242 - parser.add_argument('--block-size', type=int, default=16, choices=[1, 2, 4, 8, 16, 32, 64, 128, 256], help='token block size')
243 - # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
244 - parser.add_argument('--seed', type=int, default=0, help='random seed')
245 - parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
246 - parser.add_argument('--gpu-memory-utilization', type=float, default=0.95, help='the percentage of GPU memory to be used for the model executor')
247 - parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens per iteration')
248 - parser.add_argument('--max-num-sequences', type=int, default=256, help='maximum number of sequences per iteration')
249 - parser.add_argument('--log-stats', action='store_true', help='log system statistics')
250 - return parser
251 -
252 -
253 -def process_server_arguments(args: argparse.Namespace):
254 - """Post process the parsed arguments."""
255 - if args.pipeline_parallel_size * args.tensor_parallel_size > 1:
256 - args.use_ray = True
257 - args.swap_space = args.swap_space * _GiB
258 - args.max_num_sequences = min(args.max_num_sequences, args.max_num_batched_tokens)
259 - return args
260 -
261 -
262 -def init_local_server_and_frontend_with_arguments(args: argparse.Namespace):
263 - # TODO(zhuohan): Support pipeline parallelism.
264 - assert args.pipeline_parallel_size == 1, (
265 - 'Pipeline parallelism is not supported yet.')
266 -
267 - (num_nodes, num_devices_per_node, distributed_init_method,
268 - all_stage_devices) = (
269 - initialize_cluster(
270 - use_ray=args.use_ray,
271 - pipeline_parallel_size=args.pipeline_parallel_size,
272 - tensor_parallel_size=args.tensor_parallel_size))
273 -
274 - # Create a server.
275 - server = Server(
276 - model=args.model,
277 - cache_dir=args.cache_dir,
278 - use_dummy_weights=args.use_dummy_weights,
279 - use_np_cache=args.use_np_cache,
280 - pipeline_parallel_size=args.pipeline_parallel_size,
281 - tensor_parallel_size=args.tensor_parallel_size,
282 - block_size=args.block_size,
283 - dtype=args.dtype,
284 - seed=args.seed,
285 - swap_space=args.swap_space,
286 - gpu_memory_utilization=args.gpu_memory_utilization,
287 - max_num_batched_tokens=args.max_num_batched_tokens,
288 - max_num_sequences=args.max_num_sequences,
289 - num_nodes=num_nodes,
290 - num_devices_per_node=num_devices_per_node,
291 - distributed_init_method=distributed_init_method,
292 - all_stage_devices=all_stage_devices,
293 - use_ray=args.use_ray,
294 - log_stats=args.log_stats,
295 - )
296 -
297 - # Create a frontend.
298 - frontend = SimpleFrontend(
299 - model_name=args.model,
300 - block_size=args.block_size,
301 - )
302 - return server, frontend
cacheflow/entrypoints/fastapi_server.py new
+128
@@ -0,0 +1,128 @@
1 +import argparse
2 +import asyncio
3 +import json
4 +import time
5 +from typing import Any, Dict
6 +import uuid
7 +
8 +from fastapi import FastAPI, Request
9 +from fastapi.responses import StreamingResponse
10 +import ray
11 +import uvicorn
12 +
13 +from cacheflow.outputs import RequestOutput
14 +from cacheflow.sampling_params import SamplingParams
15 +from cacheflow.server.arg_utils import (
16 + add_server_arguments, create_server_configs_from_args)
17 +from cacheflow.server.llm_server import LLMServer
18 +from cacheflow.server.ray_utils import initialize_cluster
19 +
20 +TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
21 +app = FastAPI()
22 +
23 +
24 +class FastAPIServer:
25 +
26 + def __init__(self, server_use_ray: bool, *args, **kwargs) -> None:
27 + if server_use_ray:
28 + remote_server_class = ray.remote(num_cpus=0)(LLMServer)
29 + else:
30 + remote_server_class = ray.remote(num_gpus=1)(LLMServer)
31 + self.server = remote_server_class.remote(*args, **kwargs)
32 +
33 + # Request id -> request output.
34 + self.request_outputs: Dict[str, RequestOutput] = {}
35 + # Request id -> event to notify that there is new output.
36 + self.request_events: Dict[str, asyncio.Event] = {}
37 + self.is_server_running = False
38 +
39 + async def server_step(self):
40 + self.is_server_running = True
41 + request_outputs = await self.server.step.remote()
42 + self.is_server_running = False
43 + # Notify the waiting coroutines that there are new outputs ready.
44 + for request_output in request_outputs:
45 + request_id = request_output.request_id
46 + self.request_outputs[request_id] = request_output
47 + self.request_events[request_id].set()
48 +
49 + async def generate(self, request_dict: Dict[str, Any]):
50 + # Preprocess the request.
51 + arrival_time = time.time()
52 + prompt = request_dict.pop("prompt")
53 + sampling_params = SamplingParams(**request_dict)
54 +
55 + # Create an event to notify us that there is new output from the
56 + # cacheflow server.
57 + request_id = str(uuid.uuid4().hex[:8])
58 + request_event = asyncio.Event()
59 + self.request_events[request_id] = request_event
60 +
61 + # Add the request into the cacheflow server's waiting queue.
62 + await self.server.add_request.remote(
63 + request_id, prompt, sampling_params, arrival_time=arrival_time)
64 +
65 + # The cacheflow server does not have a background loop that keeps
66 + # processing incoming requests. Therefore, we need to keep kicking
67 + # the server to process the requests.
68 + while True:
69 + # Kick the server if the server is not running.
70 + if not self.is_server_running:
71 + await self.server_step()
72 +
73 + # Wait for new output. The group_event will be set in server_step
74 + # when there is new output available for the sequence group.
75 + # Added a timeout to prevent deadlock.
76 + try:
77 + await asyncio.wait_for(request_event.wait(),
78 + timeout=TIMEOUT_TO_PREVENT_DEADLOCK)
79 + except asyncio.TimeoutError:
80 + continue
81 + # Reset the event to wait for the next output.
82 + request_event.clear()
83 +
84 + # Decode and return new outputs.
85 + request_output = self.request_outputs[request_id]
86 + prompt = request_output.prompt
87 + text_outputs = [
88 + prompt + output.text
89 + for output in request_output.outputs
90 + ]
91 + ret = {
92 + "text": text_outputs,
93 + "error": 0,
94 + }
95 + yield (json.dumps(ret) + "\0").encode("utf-8")
96 +
97 + # Once finished, release the resources of the sequence group.
98 + if request_output.done:
99 + del self.request_outputs[request_id]
100 + del self.request_events[request_id]
101 + # Kick the server if the server is not running. This is to
102 + # prevent that there are still requests in server's waiting
103 + # queue to be executed.
104 + if not self.is_server_running:
105 + await self.server_step()
106 + break
107 +
108 +
109 +@app.post("/generate")
110 +async def generate_stream(request: Request):
111 + request_dict = await request.json()
112 + return StreamingResponse(server.generate(request_dict))
113 +
114 +
115 +if __name__ == "__main__":
116 + parser = argparse.ArgumentParser()
117 + parser.add_argument("--host", type=str, default="localhost")
118 + parser.add_argument("--port", type=int, default=10002)
119 + parser = add_server_arguments(parser)
120 + args = parser.parse_args()
121 +
122 + server_configs = create_server_configs_from_args(args)
123 + parallel_config = server_configs[2]
124 + distributed_init_method, stage_devices = initialize_cluster(parallel_config)
125 +
126 + server = FastAPIServer(
127 + args.use_ray, *server_configs, distributed_init_method, stage_devices)
128 + uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/frontend/fastapi_frontend.py deleted
-201
@@ -1,201 +0,0 @@
1 -import argparse
2 -import asyncio
3 -import json
4 -import time
5 -from typing import List, Dict, Optional
6 -
7 -from fastapi import FastAPI, Request
8 -from fastapi.responses import StreamingResponse
9 -import ray
10 -import uvicorn
11 -
12 -from cacheflow.core.server import (Server, add_server_arguments,
13 - process_server_arguments,
14 - initialize_cluster)
15 -from cacheflow.frontend.utils import get_tokenizer
16 -from cacheflow.sampling_params import SamplingParams
17 -from cacheflow.sequence import Sequence, SequenceGroup
18 -from cacheflow.utils import Counter
19 -from cacheflow.worker.controller import DeviceID
20 -
21 -TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
22 -app = FastAPI()
23 -
24 -
25 -class FastAPIServer:
26 - def __init__(
27 - self,
28 - model: str,
29 - cache_dir: Optional[str],
30 - use_np_cache: bool,
31 - pipeline_parallel_size: int,
32 - tensor_parallel_size: int,
33 - block_size: int,
34 - dtype: str,
35 - seed: int,
36 - swap_space: int,
37 - gpu_memory_utilization: float,
38 - max_num_batched_tokens: int,
39 - max_num_sequences: int,
40 - num_nodes: int,
41 - num_devices_per_node: int,
42 - distributed_init_method: str,
43 - all_stage_devices: List[List[DeviceID]],
44 - server_use_ray: bool,
45 - log_stats: bool,
46 - ):
47 - self.block_size = block_size
48 -
49 - self.tokenizer = get_tokenizer(model)
50 - self.seq_group_counter = Counter()
51 - self.seq_counter = Counter()
52 - if server_use_ray:
53 - remote_server_class = ray.remote(num_cpus=0)(Server)
54 - else:
55 - remote_server_class = ray.remote(num_gpus=1)(Server)
56 - self.server = remote_server_class.remote(
57 - model=model,
58 - cache_dir=cache_dir,
59 - use_dummy_weights=False,
60 - use_np_cache=use_np_cache,
61 - pipeline_parallel_size=pipeline_parallel_size,
62 - tensor_parallel_size=tensor_parallel_size,
63 - block_size=block_size,
64 - dtype=dtype,
65 - seed=seed,
66 - swap_space=swap_space,
67 - gpu_memory_utilization=gpu_memory_utilization,
68 - max_num_batched_tokens=max_num_batched_tokens,
69 - max_num_sequences=max_num_sequences,
70 - num_nodes=num_nodes,
71 - num_devices_per_node=num_devices_per_node,
72 - distributed_init_method=distributed_init_method,
73 - all_stage_devices=all_stage_devices,
74 - use_ray=server_use_ray,
75 - log_stats=log_stats,
76 - )
77 -
78 - self.running_seq_groups: Dict[int, SequenceGroup] = {}
79 - self.sequence_group_events: Dict[int, asyncio.Event] = {}
80 - self.is_server_running = False
81 -
82 - async def server_step(self):
83 - self.is_server_running = True
84 - updated_seq_groups = await self.server.step.remote()
85 - self.is_server_running = False
86 - # Notify the waiting coroutines that there are new outputs ready.
87 - for seq_group in updated_seq_groups:
88 - group_id = seq_group.group_id
89 - self.running_seq_groups[group_id] = seq_group
90 - self.sequence_group_events[group_id].set()
91 -
92 - async def generate(self, request_dict: Dict):
93 - # Preprocess the request.
94 - prompt = request_dict.pop("prompt")
95 - sampling_params = SamplingParams(**request_dict)
96 - sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
97 - token_ids = self.tokenizer.encode(prompt)
98 - seqs: List[Sequence] = []
99 - for _ in range(sampling_params.n):
100 - seq_id = next(self.seq_counter)
101 - seq = Sequence(seq_id, prompt, token_ids, block_size=self.block_size)
102 - seqs.append(seq)
103 -
104 - arrival_time = time.time()
105 - group_id = next(self.seq_group_counter)
106 - seq_group = SequenceGroup(group_id, seqs, arrival_time)
107 - # Create an event to notify us that there is new output from the
108 - # cacheflow server.
109 - group_event = asyncio.Event()
110 - self.running_seq_groups[group_id] = seq_group
111 - self.sequence_group_events[group_id] = group_event
112 - # Add the request into the cacheflow server's waiting queue.
113 - await self.server.add_sequence_groups.remote([(seq_group, sampling_params)])
114 - # The cacheflow server does not have a background loop that keeps
115 - # processing incoming requests. Therefore, we need to keep kicking
116 - # the server to process the requests.
117 - while True:
118 - # Kick the server if the server is not running.
119 - if not self.is_server_running:
120 - await self.server_step()
121 - # Wait for new output. The group_event will be set in server_step
122 - # when there is new output available for the sequence group.
123 - # Added a timeout to prevent deadlock.
124 - try:
125 - await asyncio.wait_for(group_event.wait(), timeout=TIMEOUT_TO_PREVENT_DEADLOCK)
126 - except asyncio.TimeoutError:
127 - continue
128 - # Reset the event to wait for the next output.
129 - group_event.clear()
130 - # Decode and return new outputs
131 - seq_group = self.running_seq_groups[group_id]
132 - all_outputs = []
133 - for seq in seq_group.seqs:
134 - token_ids = seq.get_token_ids()
135 - output = self.tokenizer.decode(token_ids, skip_special_tokens=True)
136 - all_outputs.append(output)
137 - ret = {
138 - "text": all_outputs,
139 - "error": 0,
140 - }
141 - yield (json.dumps(ret) + "\0").encode("utf-8")
142 -
143 - # Once finished, release the resources of the sequence group.
144 - if seq_group.is_finished():
145 - del self.running_seq_groups[group_id]
146 - del self.sequence_group_events[group_id]
147 - # Kick the server if the server is not running. This is to
148 - # prevent that there are still requests in server's waiting
149 - # queue to be executed.
150 - if not self.is_server_running:
151 - await self.server_step()
152 - break
153 -
154 -
155 -@app.post("/generate")
156 -async def generate_stream(request: Request):
157 - request_dict = await request.json()
158 - return StreamingResponse(server.generate(request_dict))
159 -
160 -
161 -if __name__ == "__main__":
162 - parser = argparse.ArgumentParser()
163 - parser.add_argument("--host", type=str, default="localhost")
164 - parser.add_argument("--port", type=int, default=10002)
165 - parser = add_server_arguments(parser)
166 - args = parser.parse_args()
167 - args = process_server_arguments(args)
168 -
169 - # TODO(zhuohan): Support pipeline parallelism.
170 - assert args.pipeline_parallel_size == 1, (
171 - 'Pipeline parallelism is not supported yet.')
172 -
173 - (num_nodes, num_devices_per_node, distributed_init_method,
174 - all_stage_devices) = (
175 - initialize_cluster(
176 - use_ray=True,
177 - pipeline_parallel_size=args.pipeline_parallel_size,
178 - tensor_parallel_size=args.tensor_parallel_size))
179 -
180 - server = FastAPIServer(
181 - model=args.model,
182 - cache_dir=args.cache_dir,
183 - use_np_cache=args.use_np_cache,
184 - pipeline_parallel_size=args.pipeline_parallel_size,
185 - tensor_parallel_size=args.tensor_parallel_size,
186 - block_size=args.block_size,
187 - dtype=args.dtype,
188 - seed=args.seed,
189 - swap_space=args.swap_space,
190 - gpu_memory_utilization=args.gpu_memory_utilization,
191 - max_num_batched_tokens=args.max_num_batched_tokens,
192 - max_num_sequences=args.max_num_sequences,
193 - num_nodes=num_nodes,
194 - num_devices_per_node=num_devices_per_node,
195 - distributed_init_method=distributed_init_method,
196 - all_stage_devices=all_stage_devices,
197 - server_use_ray=args.use_ray,
198 - log_stats=args.log_stats,
199 - )
200 -
201 - uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/frontend/simple_frontend.py deleted
-72
@@ -1,72 +0,0 @@
1 -import time
2 -from typing import List, Optional, Tuple
3 -
4 -from cacheflow.frontend.utils import get_tokenizer
5 -from cacheflow.logger import init_logger
6 -from cacheflow.sampling_params import SamplingParams
7 -from cacheflow.sequence import Sequence, SequenceGroup
8 -from cacheflow.utils import Counter
9 -
10 -logger = init_logger(__name__)
11 -
12 -
13 -class SimpleFrontend:
14 -
15 - def __init__(
16 - self,
17 - model_name: str,
18 - block_size: int,
19 - ) -> None:
20 - self.block_size = block_size
21 -
22 - self.tokenizer = get_tokenizer(model_name)
23 - self.seq_group_counter = Counter()
24 - self.seq_counter = Counter()
25 - self.inputs: List[Tuple[SequenceGroup, SamplingParams]] = []
26 -
27 - def add_eos_token(self, sampling_params: SamplingParams) -> SamplingParams:
28 - # Stop generation when we see an EOS token.
29 - sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
30 - return sampling_params
31 -
32 - def query(
33 - self,
34 - prompt: str,
35 - sampling_params: SamplingParams,
36 - ) -> None:
37 - token_ids = self.tokenizer.encode(prompt)
38 - self._add_query(prompt, token_ids, sampling_params)
39 -
40 - def _add_query(
41 - self,
42 - prompt: str,
43 - token_ids: List[int],
44 - sampling_params: SamplingParams,
45 - arrival_time: Optional[float] = None,
46 - ) -> None:
47 - if arrival_time is None:
48 - arrival_time = time.time()
49 - seqs: List[Sequence] = []
50 - for _ in range(sampling_params.n):
51 - seq_id = next(self.seq_counter)
52 - seq = Sequence(seq_id, prompt, token_ids, block_size=self.block_size)
53 - seqs.append(seq)
54 -
55 - group_id = next(self.seq_group_counter)
56 - seq_group = SequenceGroup(group_id, seqs, arrival_time)
57 - self.inputs.append((seq_group, sampling_params))
58 -
59 - def get_inputs(self) -> List[Tuple[SequenceGroup, SamplingParams]]:
60 - inputs = self.inputs
61 - self.inputs = []
62 - return inputs
63 -
64 - def print_response(
65 - self,
66 - seq_group: SequenceGroup,
67 - ) -> None:
68 - for seq in seq_group.seqs:
69 - token_ids = seq.get_token_ids()
70 - output = self.tokenizer.decode(token_ids, skip_special_tokens=True)
71 - output = output.strip()
72 - logger.info(f"Seq {seq.seq_id}: {output!r}")
cacheflow/model_executor/__init__.py
+1 -3
@@ -1,12 +1,10 @@
1 from cacheflow.model_executor.input_metadata import InputMetadata
2 from cacheflow.model_executor.model_loader import get_model
3 -from cacheflow.model_executor.utils import (set_random_seed,
4 - get_cache_block_size)
3 +from cacheflow.model_executor.utils import set_random_seed
4
5
6 __all__ = [
7 "InputMetadata",
9 - "get_cache_block_size",
8 "get_model",
9 "set_random_seed",
10 ]
cacheflow/model_executor/layers/attention.py
+1 -1
@@ -10,9 +10,9 @@ from cacheflow import cache_ops
10 from cacheflow import pos_encoding_ops
11 from cacheflow.model_executor.input_metadata import InputMetadata
12
13 -
13 _SUPPORTED_HEAD_SIZES = [32, 64, 80, 96, 128, 160, 192, 256]
14
15 +
16 class GPTCacheFlowAttention(nn.Module):
17 """GPT-style multi-head attention.
18
cacheflow/model_executor/model_loader.py
+12 -43
@@ -1,16 +1,13 @@
1 """Utilities for selecting and loading models."""
2 -from typing import Optional
3 -
2 import torch
3 import torch.nn as nn
6 -from transformers import AutoConfig, PretrainedConfig
4 +from transformers import PretrainedConfig
5
6 +from cacheflow.config import ModelConfig
7 from cacheflow.model_executor.models import (
8 GPT2LMHeadModel, GPTNeoXForCausalLM, LlamaForCausalLM, OPTForCausalLM)
10 -from cacheflow.model_executor.utils import get_torch_dtype
9 from cacheflow.model_executor.weight_utils import initialize_dummy_weights
10
13 -
11 # TODO(woosuk): Lazy-load the model classes.
12 _MODEL_REGISTRY = {
13 "GPT2LMHeadModel": GPT2LMHeadModel,
@@ -19,6 +16,7 @@ _MODEL_REGISTRY = {
16 "OPTForCausalLM": OPTForCausalLM,
17 }
18
19 +
20 def _get_model_architecture(config: PretrainedConfig) -> nn.Module:
21 architectures = getattr(config, "architectures", [])
22 for arch in architectures:
@@ -30,51 +28,22 @@ def _get_model_architecture(config: PretrainedConfig) -> nn.Module:
28 )
29
30
33 -def _get_dtype(config: PretrainedConfig, dtype: str) -> torch.dtype:
34 - # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
35 - # because config.torch_dtype can be None.
36 - config_dtype = getattr(config, "torch_dtype", None)
37 - if config_dtype is None:
38 - config_dtype = torch.float32
39 - if dtype == "default":
40 - if config_dtype == torch.float32:
41 - # Following the common practice, we use float16 for float32 models.
42 - torch_dtype = torch.float16
43 - else:
44 - torch_dtype = config_dtype
45 - else:
46 - torch_dtype = get_torch_dtype(dtype)
47 - if torch_dtype != config_dtype and config_dtype != torch.float32:
48 - # TODO(woosuk): Allow using float16 for bfloat16 models and
49 - # vice versa. Print a warning message and continue.
50 - raise ValueError(
51 - f"Cannot use {torch_dtype} for {config_dtype} model.")
52 - return torch_dtype
53 -
54 -
55 -def get_model(
56 - model_name: str,
57 - dtype: str,
58 - cache_dir: Optional[str],
59 - use_dummy_weights: bool,
60 - use_np_cache: bool,
61 -) -> nn.Module:
62 - config = AutoConfig.from_pretrained(model_name)
63 - torch_dtype = _get_dtype(config, dtype)
64 - torch.set_default_dtype(torch_dtype)
65 - model_class = _get_model_architecture(config)
31 +def get_model(model_config: ModelConfig) -> nn.Module:
32 + model_class = _get_model_architecture(model_config.hf_config)
33 + torch.set_default_dtype(model_config.dtype)
34
35 # Create a model instance.
36 # The weights will be initialized as empty tensors.
69 - model = model_class(config)
70 - if use_dummy_weights:
37 + model = model_class(model_config.hf_config)
38 + if model_config.use_dummy_weights:
39 model = model.cuda()
40 # NOTE(woosuk): For accurate performance evaluation, we assign
41 # random values to the weights.
42 initialize_dummy_weights(model)
43 else:
44 # Load the weights from the cached or downloaded files.
77 - model.load_weights(model_name, cache_dir, use_np_cache)
45 + model.load_weights(
46 + model_config.model, model_config.download_dir,
47 + model_config.use_np_weights)
48 model = model.cuda()
79 - return model.eval(), torch_dtype
80 -
49 + return model.eval()
cacheflow/model_executor/utils.py
-35
@@ -1,6 +1,5 @@
1 """Utils for model executor."""
2 import random
3 -from typing import Union
3
4 import numpy as np
5 import torch
@@ -9,28 +8,6 @@ from cacheflow.model_executor.parallel_utils.parallel_state import model_paralle
8 from cacheflow.model_executor.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
9
10
12 -_STR_DTYPE_TO_TORCH_DTYPE = {
13 - "half": torch.half,
14 - "float": torch.float,
15 - "float16": torch.float16,
16 - "float32": torch.float32,
17 - "bfloat16": torch.bfloat16,
18 -}
19 -
20 -
21 -def get_torch_dtype(dtype: Union[torch.dtype, str]) -> torch.dtype:
22 - if isinstance(dtype, str):
23 - torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
24 - else:
25 - torch_dtype = dtype
26 - return torch_dtype
27 -
28 -
29 -def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
30 - torch_dtype = get_torch_dtype(dtype)
31 - return torch.tensor([], dtype=torch_dtype).element_size()
32 -
33 -
11 def set_random_seed(seed: int) -> None:
12 random.seed(seed)
13 np.random.seed(seed)
@@ -40,15 +17,3 @@ def set_random_seed(seed: int) -> None:
17
18 if model_parallel_is_initialized():
19 model_parallel_cuda_manual_seed(seed)
43 -
44 -
45 -def get_cache_block_size(block_size: int,
46 - num_heads: int,
47 - head_size: int,
48 - num_layers: int,
49 - dtype: str) -> int:
50 - key_cache_block = block_size * num_heads * head_size
51 - value_cache_block = key_cache_block
52 - total = num_layers * (key_cache_block + value_cache_block)
53 - dtype_size = get_dtype_size(dtype)
54 - return dtype_size * total
cacheflow/outputs.py new
+79
@@ -0,0 +1,79 @@
1 +from typing import Dict, List, Union
2 +
3 +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
4 +
5 +from cacheflow.sequence import SequenceGroup
6 +
7 +
8 +class CompletionOutput:
9 +
10 + def __init__(
11 + self,
12 + text: str,
13 + token_ids: List[int],
14 + cumulative_logprobs: float,
15 + logprobs: List[Dict[int, float]],
16 + ) -> None:
17 + self.text = text
18 + self.token_ids = token_ids
19 + self.cumulative_logprobs = cumulative_logprobs
20 + self.logprobs = logprobs
21 +
22 + def __repr__(self) -> str:
23 + return (f"CompletionOutput(output={self.text!r}, "
24 + f"token_ids={self.token_ids}, "
25 + f"cumulative_logprobs={self.cumulative_logprobs}, "
26 + f"logprobs={self.logprobs})")
27 +
28 +
29 +class RequestOutput:
30 +
31 + def __init__(
32 + self,
33 + request_id: int,
34 + prompt: str,
35 + prompt_token_ids: List[int],
36 + outputs: List[CompletionOutput],
37 + done: bool = False,
38 + ) -> None:
39 + self.request_id = request_id
40 + self.prompt = prompt
41 + self.prompt_token_ids = prompt_token_ids
42 + self.outputs = outputs
43 + self.done = done
44 +
45 + @staticmethod
46 + def from_seq_group(
47 + seq_group: SequenceGroup,
48 + tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
49 + ) -> "RequestOutput":
50 + outputs: List[CompletionOutput] = []
51 + seqs = seq_group.get_seqs()
52 + for seq in seqs:
53 + output_token_ids = seq.data.output_token_ids
54 + output_str = tokenizer.decode(output_token_ids,
55 + skip_special_tokens=True)
56 + seq_logprobs = seq.data.cumulative_logprobs
57 +
58 + logprobs = seq.output_logprobs
59 + if seq_group.sampling_params.logprobs == 0:
60 + # NOTE: We need to take care of this case because the sequence
61 + # always has the logprobs of the sampled tokens even if the
62 + # logprobs are not requested.
63 + logprobs = {}
64 + output = CompletionOutput(output_str, output_token_ids,
65 + seq_logprobs, logprobs)
66 + outputs.append(output)
67 +
68 + # Every sequence in the sequence group should have the same prompt.
69 + prompt = seqs[0].prompt
70 + prompt_token_ids = seqs[0].data.prompt_token_ids
71 + return RequestOutput(seq_group.request_id, prompt, prompt_token_ids,
72 + outputs, seq_group.is_finished())
73 +
74 + def __repr__(self) -> str:
75 + return (f"RequestOutput(request_id={self.request_id}, "
76 + f"prompt={self.prompt!r}, "
77 + f"prompt_token_ids={self.prompt_token_ids}, "
78 + f"outputs={self.outputs}, "
79 + f"done={self.done})")
cacheflow/sampling_params.py
+1 -1
@@ -116,4 +116,4 @@ class SamplingParams:
116 f"use_beam_search={self.use_beam_search}, "
117 f"stop_token_ids={self.stop_token_ids}, "
118 f"max_tokens={self.max_tokens}, "
119 - f"logprobs={self.logprobs}")
119 + f"logprobs={self.logprobs})")
cacheflow/sequence.py
+9 -6
@@ -115,12 +115,14 @@ class SequenceGroup:
115
116 def __init__(
117 self,
118 - group_id: int,
118 + request_id: str,
119 seqs: List[Sequence],
120 + sampling_params: SamplingParams,
121 arrival_time: float,
122 ) -> None:
122 - self.group_id = group_id
123 + self.request_id = request_id
124 self.seqs = seqs
125 + self.sampling_params = sampling_params
126 self.arrival_time = arrival_time
127
128 def get_seqs(
@@ -145,21 +147,22 @@ class SequenceGroup:
147 return all(seq.status == SequenceStatus.FINISHED for seq in self.seqs)
148
149 def __repr__(self) -> str:
148 - return (f'SequenceGroup(group_id={self.group_id}, '
149 - f'num_seqs={len(self.seqs)})')
150 + return (f"SequenceGroup(request_id={self.request_id}, "
151 + f"sampling_params={self.sampling_params}, "
152 + f"num_seqs={len(self.seqs)})")
153
154
155 class SequenceGroupMetadata:
156
157 def __init__(
158 self,
156 - group_id: int,
159 + request_id: str,
160 is_prompt: bool,
161 seq_data: Dict[int, SequenceData], # Seq id -> sequence data.
162 sampling_params: SamplingParams,
163 block_tables: Dict[int, List[int]], # Seq id -> list of physical block numbers.
164 ) -> None:
162 - self.group_id = group_id
165 + self.request_id = request_id
166 self.is_prompt = is_prompt
167 self.seq_data = seq_data
168 self.sampling_params = sampling_params
cacheflow/server/arg_utils.py new
+74
@@ -0,0 +1,74 @@
1 +import argparse
2 +from typing import Tuple
3 +
4 +from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
5 + SchedulerConfig)
6 +from cacheflow.server.llm_server import LLMServer
7 +from cacheflow.server.ray_utils import initialize_cluster
8 +
9 +_GiB = 1 << 30
10 +
11 +
12 +def add_server_arguments(parser: argparse.ArgumentParser):
13 + """Shared arguments for CacheFlow servers."""
14 + # Model arguments
15 + parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
16 + parser.add_argument('--download-dir', type=str, default=None,
17 + help='directory to download and load the weights, '
18 + 'default to the default cache dir of huggingface')
19 + parser.add_argument('--use-np-weights', action='store_true',
20 + help='save a numpy copy of model weights for faster loading')
21 + parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
22 + # TODO(woosuk): Support FP32.
23 + parser.add_argument('--dtype', type=str, default='default', choices=['default', 'half', 'bfloat16'],
24 + help=('data type for model weights and activations. '
25 + 'The "default" option will use FP16 precision '
26 + 'for FP32 and FP16 models, and BF16 precision '
27 + 'for BF16 models.'))
28 + # Parallel arguments
29 + parser.add_argument('--use-ray', action='store_true', help='use Ray for distributed serving, will be automatically set when using more than 1 GPU')
30 + parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
31 + parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
32 + # KV cache arguments
33 + parser.add_argument('--block-size', type=int, default=16, choices=[1, 2, 4, 8, 16, 32, 64, 128, 256], help='token block size')
34 + # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
35 + parser.add_argument('--seed', type=int, default=0, help='random seed')
36 + parser.add_argument('--swap-space', type=int, default=4, help='CPU swap space size (GiB) per GPU')
37 + parser.add_argument('--gpu-memory-utilization', type=float, default=0.95, help='the percentage of GPU memory to be used for the model executor')
38 + parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens per iteration')
39 + parser.add_argument('--max-num-seqs', type=int, default=256, help='maximum number of sequences per iteration')
40 + parser.add_argument('--disable-log-stats', action='store_true', help='disable logging statistics')
41 + return parser
42 +
43 +
44 +def create_server_configs_from_args(
45 + args: argparse.Namespace,
46 +) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
47 + # Post-process the parsed arguments.
48 + args.swap_space = args.swap_space * _GiB
49 + args.max_num_seqs = min(args.max_num_seqs, args.max_num_batched_tokens)
50 +
51 + # Initialize the configs.
52 + model_config = ModelConfig(
53 + args.model, args.download_dir, args.use_np_weights,
54 + args.use_dummy_weights, args.dtype, args.seed)
55 + cache_config = CacheConfig(args.block_size, args.gpu_memory_utilization,
56 + args.swap_space)
57 + parallel_config = ParallelConfig(args.pipeline_parallel_size,
58 + args.tensor_parallel_size, args.use_ray)
59 + scheduler_config = SchedulerConfig(args.max_num_batched_tokens,
60 + args.max_num_seqs)
61 + return model_config, cache_config, parallel_config, scheduler_config
62 +
63 +
64 +def initialize_server_from_args(args: argparse.Namespace) -> LLMServer:
65 + server_configs = create_server_configs_from_args(args)
66 + parallel_config = server_configs[2]
67 +
68 + # Initialize the cluster.
69 + distributed_init_method, devices = initialize_cluster(parallel_config)
70 +
71 + # Create the LLM server.
72 + server = LLMServer(*server_configs, distributed_init_method, devices,
73 + log_stats=not args.disable_log_stats)
74 + return server
cacheflow/server/llm_server.py new
+198
@@ -0,0 +1,198 @@
1 +import time
2 +from typing import Any, List, Optional
3 +
4 +try:
5 + import ray
6 +except ImportError:
7 + ray = None
8 +
9 +from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
10 + SchedulerConfig)
11 +from cacheflow.core.scheduler import Scheduler
12 +from cacheflow.logger import init_logger
13 +from cacheflow.outputs import RequestOutput
14 +from cacheflow.sampling_params import SamplingParams
15 +from cacheflow.server.tokenizer_utils import get_tokenizer
16 +from cacheflow.sequence import Sequence, SequenceGroup
17 +from cacheflow.utils import Counter
18 +from cacheflow.worker.worker import Worker
19 +
20 +logger = init_logger(__name__)
21 +
22 +
23 +class LLMServer:
24 +
25 + def __init__(
26 + self,
27 + model_config: ModelConfig,
28 + cache_config: CacheConfig,
29 + parallel_config: ParallelConfig,
30 + scheduler_config: SchedulerConfig,
31 + distributed_init_method: str,
32 + stage_devices: List[List[Any]],
33 + log_stats: bool = True,
34 + ) -> None:
35 + logger.info(
36 + "Initializing an LLM server with config: "
37 + f"model={model_config.model!r}, "
38 + f"dtype={model_config.dtype}, "
39 + f"use_dummy_weights={model_config.use_dummy_weights}, "
40 + f"download_dir={model_config.download_dir!r}, "
41 + f"use_np_weights={model_config.use_np_weights}, "
42 + f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
43 + f"seed={model_config.seed})"
44 + )
45 + # TODO(woosuk): Print more configs in debug mode.
46 +
47 + self.model_config = model_config
48 + self.cache_config = cache_config
49 + self.parallel_config = parallel_config
50 + self.scheduler_config = scheduler_config
51 + self.log_stats = log_stats
52 +
53 + self._verify_args()
54 +
55 + self.tokenizer = get_tokenizer(model_config.model)
56 + self.seq_counter = Counter()
57 +
58 + # Create the parallel GPU workers.
59 + self.workers: List[Worker] = []
60 + assert len(stage_devices) == 1, "Only support one stage for now."
61 + for rank, node_resource, _ in stage_devices[0]:
62 + worker_cls = Worker
63 + if self.parallel_config.use_ray:
64 + worker_cls = ray.remote(
65 + num_cpus=0,
66 + num_gpus=1,
67 + resources={node_resource: 1e-5},
68 + )(worker_cls).remote
69 +
70 + worker = worker_cls(
71 + model_config,
72 + parallel_config,
73 + scheduler_config,
74 + rank,
75 + distributed_init_method,
76 + )
77 + self.workers.append(worker)
78 + # Profile the memory usage and initialize the cache.
79 + self._init_cache()
80 +
81 + # Create the scheduler.
82 + self.scheduler = Scheduler(scheduler_config, cache_config, log_stats)
83 +
84 + def _verify_args(self) -> None:
85 + self.model_config.verify_with_parallel_config(self.parallel_config)
86 +
87 + def _init_cache(self) -> None:
88 + # Get the maximum number of blocks that can be allocated on GPU and CPU.
89 + num_blocks = self._run_workers(
90 + "profile_num_available_blocks",
91 + get_all_outputs=True,
92 + block_size=self.cache_config.block_size,
93 + gpu_memory_utilization=self.cache_config.gpu_memory_utilization,
94 + cpu_swap_space=self.cache_config.swap_space,
95 + )
96 +
97 + # Since we use a shared centralized controller, we take the minimum
98 + # number of blocks across all workers to make sure all the memory
99 + # operators can be applied to all workers.
100 + num_gpu_blocks = min(b[0] for b in num_blocks)
101 + num_cpu_blocks = min(b[1] for b in num_blocks)
102 + # FIXME(woosuk): Change to debug log.
103 + logger.info(f'# GPU blocks: {num_gpu_blocks}, '
104 + f'# CPU blocks: {num_cpu_blocks}')
105 + self.cache_config.num_gpu_blocks = num_gpu_blocks
106 + self.cache_config.num_cpu_blocks = num_cpu_blocks
107 +
108 + # Initialize the cache.
109 + self._run_workers("init_cache_engine", cache_config=self.cache_config)
110 +
111 + def add_request(
112 + self,
113 + request_id: str,
114 + prompt: str,
115 + sampling_params: SamplingParams,
116 + prompt_token_ids: Optional[List[int]] = None,
117 + arrival_time: Optional[float] = None,
118 + ) -> None:
119 + if arrival_time is None:
120 + arrival_time = time.time()
121 + if prompt_token_ids is None:
122 + prompt_token_ids = self.tokenizer.encode(prompt)
123 +
124 + # Create the sequences.
125 + block_size = self.cache_config.block_size
126 + seqs: List[Sequence] = []
127 + for _ in range(sampling_params.n):
128 + seq_id = next(self.seq_counter)
129 + seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
130 + seqs.append(seq)
131 +
132 + # FIXME(woosuk)
133 + # Add the EOS token to the stop token list.
134 + sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
135 +
136 + # Create the sequence group.
137 + seq_group = SequenceGroup(request_id, seqs, sampling_params,
138 + arrival_time)
139 +
140 + # Add the sequence group to the scheduler.
141 + self.scheduler.add_seq_group(seq_group)
142 +
143 + def has_unfinished_requests(self) -> bool:
144 + return self.scheduler.has_unfinished_seqs()
145 +
146 + def step(self) -> List[RequestOutput]:
147 + seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
148 + if (not seq_group_metadata_list) and scheduler_outputs.is_empty():
149 + # Nothing to do.
150 + return []
151 +
152 + # Execute the model.
153 + output = self._run_workers(
154 + "execute_model",
155 + seq_group_metadata_list=seq_group_metadata_list,
156 + blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
157 + blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
158 + blocks_to_copy=scheduler_outputs.blocks_to_copy,
159 + )
160 + # Update the scheduler.
161 + updated_seq_groups = self.scheduler.update(output)
162 +
163 + # Create the outputs.
164 + request_outputs: List[RequestOutput] = []
165 + for seq_group in updated_seq_groups:
166 + # TODO(woosuk): Batch-decode the outputs for speedup.
167 + request_output = RequestOutput.from_seq_group(seq_group,
168 + self.tokenizer)
169 + request_outputs.append(request_output)
170 + return request_outputs
171 +
172 + def _run_workers(
173 + self,
174 + method: str,
175 + get_all_outputs: bool = False,
176 + *args,
177 + **kwargs,
178 + ) -> Any:
179 + all_outputs = []
180 + for worker in self.workers:
181 + executor = getattr(worker, method)
182 + if self.parallel_config.use_ray:
183 + executor = executor.remote
184 +
185 + output = executor(*args, **kwargs)
186 + all_outputs.append(output)
187 +
188 + if self.parallel_config.use_ray:
189 + all_outputs = ray.get(all_outputs)
190 +
191 + if get_all_outputs:
192 + return all_outputs
193 +
194 + # Make sure all workers have the same results.
195 + output = all_outputs[0]
196 + for other_output in all_outputs[1:]:
197 + assert output == other_output
198 + return output
cacheflow/server/ray_utils.py new
+90
@@ -0,0 +1,90 @@
1 +import random
2 +from typing import List, Optional, Tuple
3 +
4 +try:
5 + import ray
6 +except ImportError:
7 + ray = None
8 +
9 +from cacheflow.config import ParallelConfig
10 +
11 +DeviceID = Tuple[int, str, int] # rank, node resource (node IP), device id
12 +
13 +
14 +def initialize_cluster(
15 + parallel_config: ParallelConfig,
16 + address: Optional[str] = None,
17 +) -> Tuple[str, List[List[DeviceID]]]:
18 + if not parallel_config.use_ray:
19 + # Initialize cluster locally.
20 + port = random.randint(10000, 20000)
21 + # We need to setup the distributed init method to make sure
22 + # the distributed megatron code (e.g., get world size) works correctly.
23 + distributed_init_method = f"tcp://localhost:{port}"
24 + all_stage_devices = [[(0, None, 0)]]
25 + return distributed_init_method, all_stage_devices
26 +
27 + if ray is None:
28 + raise ImportError(
29 + "Ray is not installed. Please install Ray to use distributed "
30 + "serving.")
31 + # Connect to a ray cluster.
32 + ray.init(address=address)
33 +
34 + # Assume we have a uniform cluster that each node has the same number of
35 + # GPUs for now.
36 + valid_node_resources = []
37 + num_devices_per_node = None
38 + for node in ray.nodes():
39 + if (not node['Alive']) or node['Resources']['GPU'] <= 0:
40 + continue
41 + if num_devices_per_node is None:
42 + num_devices_per_node = node['Resources']['GPU']
43 + else:
44 + assert num_devices_per_node == node['Resources']['GPU'], (
45 + "The number of GPUs per node is not uniform.")
46 + for key in node['Resources']:
47 + if key.startswith('node:'):
48 + valid_node_resources.append(key)
49 +
50 + # Verify the parallel config.
51 + num_nodes = len(valid_node_resources)
52 + if parallel_config.world_size > num_nodes * num_devices_per_node:
53 + raise ValueError(
54 + "The number of required GPUs exceeds the total number of "
55 + "available GPUs.")
56 + if parallel_config.tensor_parallel_size >= num_devices_per_node:
57 + if parallel_config.tensor_parallel_size % num_devices_per_node != 0:
58 + raise ValueError(
59 + "The number of tensor parallelism is not divisible by the "
60 + "number of GPUs per node.")
61 + else:
62 + if num_devices_per_node % parallel_config.tensor_parallel_size != 0:
63 + raise ValueError(
64 + "The number of GPUs per node is not divisible by the number "
65 + "of tensor parallelism.")
66 +
67 + # Assign GPUs to pipeline stages.
68 + rank = 0
69 + current_node_id = 0
70 + current_device_id = 0
71 + distributed_init_method = None
72 + all_stage_devices = []
73 +
74 + for _ in range(parallel_config.pipeline_parallel_size):
75 + stage_devices = []
76 + for _ in range(parallel_config.tensor_parallel_size):
77 + node_resource = valid_node_resources[current_node_id]
78 + stage_devices.append((rank, node_resource, current_device_id))
79 + if distributed_init_method is None:
80 + ip = node_resource.split("node:")[-1]
81 + port = random.randint(10000, 20000)
82 + distributed_init_method = f"tcp://{ip}:{port}"
83 + rank += 1
84 + current_device_id += 1
85 + if current_device_id >= num_devices_per_node:
86 + current_node_id += 1
87 + current_device_id = 0
88 + all_stage_devices.append(stage_devices)
89 +
90 + return distributed_init_method, all_stage_devices
cacheflow/server/tokenizer_utils.py renamed
-1
@@ -3,7 +3,6 @@ from typing import Union
3 from transformers import (AutoConfig, AutoTokenizer, PreTrainedTokenizer,
4 PreTrainedTokenizerFast)
5
6 -
6 _MODEL_TYPES_WITH_SLOW_TOKENIZER = [
7 # LLaMA fast tokenizer has a bug related to protobuf.
8 # See https://github.com/WoosukKwon/cacheflow/issues/80#issue-1698550554
cacheflow/worker/cache_engine.py
+37 -21
@@ -4,6 +4,7 @@ from typing import Dict, List, Tuple
4 import torch
5
6 from cacheflow import cache_ops
7 +from cacheflow.config import CacheConfig, ModelConfig, ParallelConfig
8
9 KVCache = Tuple[torch.Tensor, torch.Tensor]
10
@@ -18,27 +19,22 @@ class CacheEngine:
19
20 def __init__(
21 self,
21 - worker_id: int,
22 - num_layers: int,
23 - num_heads: int,
24 - head_size: int,
25 - block_size: int,
26 - num_gpu_blocks: int,
27 - num_cpu_blocks: int,
28 - dtype: torch.dtype,
22 + cache_config: CacheConfig,
23 + model_config: ModelConfig,
24 + parallel_config: ParallelConfig,
25 ) -> None:
30 - if head_size % 16 != 0:
31 - raise ValueError(
32 - f'head_size ({head_size}) must be a multiple of 16.')
33 -
34 - self.worker_id = worker_id
35 - self.num_layers = num_layers
36 - self.num_heads = num_heads
37 - self.head_size = head_size
38 - self.block_size = block_size
39 - self.num_gpu_blocks = num_gpu_blocks
40 - self.num_cpu_blocks = num_cpu_blocks
41 - self.dtype = dtype
26 + self.cache_config = cache_config
27 + self.model_config = model_config
28 + self.parallel_config = parallel_config
29 +
30 + self.head_size = model_config.get_head_size()
31 + self.num_layers = model_config.get_num_layers(parallel_config)
32 + self.num_heads = model_config.get_num_heads(parallel_config)
33 + self.dtype = model_config.dtype
34 +
35 + self.block_size = cache_config.block_size
36 + self.num_gpu_blocks = cache_config.num_gpu_blocks
37 + self.num_cpu_blocks = cache_config.num_cpu_blocks
38
39 # Initialize the cache.
40 self.gpu_cache = self.allocate_gpu_cache()
@@ -48,7 +44,7 @@ class CacheEngine:
44 self.cache_stream = torch.cuda.Stream()
45 assert self.cache_stream != torch.cuda.current_stream()
46 # Initialize the events for stream synchronization.
51 - self.events = [torch.cuda.Event() for _ in range(num_layers)]
47 + self.events = [torch.cuda.Event() for _ in range(self.num_layers)]
48
49 def get_key_block_shape(self) -> Tuple[int, int, int, int]:
50 element_size = torch.tensor([], dtype=self.dtype).element_size()
@@ -133,3 +129,23 @@ class CacheEngine:
129 value_caches = [value_cache for _, value_cache in self.gpu_cache]
130 # NOTE(woosuk): This operation implicitly synchronizes the CPU and GPU.
131 cache_ops.copy_blocks(key_caches, value_caches, src_to_dsts)
132 +
133 + @staticmethod
134 + def get_cache_block_size(
135 + block_size: int,
136 + model_config: ModelConfig,
137 + parallel_config: ParallelConfig,
138 + ) -> int:
139 + head_size = model_config.get_head_size()
140 + num_heads = model_config.get_num_heads(parallel_config)
141 + num_layers = model_config.get_num_layers(parallel_config)
142 +
143 + key_cache_block = block_size * num_heads * head_size
144 + value_cache_block = key_cache_block
145 + total = num_layers * (key_cache_block + value_cache_block)
146 + dtype_size = _get_dtype_size(model_config.dtype)
147 + return dtype_size * total
148 +
149 +
150 +def _get_dtype_size(dtype: torch.dtype) -> int:
151 + return torch.tensor([], dtype=dtype).element_size()
cacheflow/worker/controller.py deleted
-130
@@ -1,130 +0,0 @@
1 -from typing import List, Optional, Tuple, Union
2 -
3 -try:
4 - import ray
5 -except ImportError:
6 - ray = None
7 -
8 -from cacheflow.core.scheduler import Scheduler
9 -from cacheflow.worker.worker import Worker
10 -
11 -
12 -DeviceID = Tuple[int, str, int] # rank, node resource (node IP), device id
13 -
14 -
15 -class Controller:
16 -
17 - def __init__(
18 - self,
19 - stage_id: int,
20 - stage_devices: List[DeviceID],
21 - world_size: int,
22 - tensor_parallel_size: int,
23 - pipeline_parallel_size: int,
24 - distributed_init_method: str,
25 - model_name: str,
26 - dtype: str,
27 - seed: int,
28 - cache_dir: Optional[str],
29 - use_dummy_weights: bool,
30 - use_np_cache: bool,
31 - max_num_batched_tokens: int,
32 - max_num_sequences: int,
33 - use_ray: bool,
34 - ) -> None:
35 - self.stage_id = stage_id
36 - self.stage_devices = stage_devices
37 - self.model_name = model_name
38 - self.use_ray = use_ray
39 -
40 - # Which pipeline stage is this node assigned to?
41 - self.is_first_stage = stage_id == 0
42 - self.is_last_stage = False
43 -
44 - self.workers: List[Worker] = []
45 - for rank, node_resource, device_id in stage_devices:
46 - if self.use_ray:
47 - worker_cls = ray.remote(num_cpus=0,
48 - num_gpus=1,
49 - resources={node_resource: 1e-5})(Worker).remote
50 - else:
51 - worker_cls = Worker
52 - worker = worker_cls(
53 - model_name=model_name,
54 - dtype=dtype,
55 - seed=seed,
56 - distributed_init_method=distributed_init_method,
57 - rank=rank,
58 - world_size=world_size,
59 - tensor_parallel_size=tensor_parallel_size,
60 - pipeline_parallel_size=pipeline_parallel_size,
61 - cache_dir=cache_dir,
62 - use_dummy_weights=use_dummy_weights,
63 - use_np_cache=use_np_cache,
64 - max_num_batched_tokens=max_num_batched_tokens,
65 - max_num_sequences=max_num_sequences,
66 - )
67 - self.workers.append(worker)
68 -
69 - def get_num_available_blocks(self, block_size: int, cpu_swap_space: int,
70 - gpu_memory_utilization: float) -> List[Tuple[int, int]]:
71 - all_worker_results = []
72 - for worker in self.workers:
73 - executor = worker.get_num_available_blocks
74 - if self.use_ray:
75 - executor = executor.remote
76 -
77 - result = executor(
78 - block_size,
79 - cpu_swap_space,
80 - gpu_memory_utilization,
81 - )
82 - all_worker_results.append(result)
83 - if self.use_ray:
84 - all_worker_results = ray.get(all_worker_results)
85 - return all_worker_results
86 -
87 - def init_cache_engine(self, block_size: int, num_gpu_blocks: int,
88 - num_cpu_blocks: int):
89 - all_worker_futures = []
90 - for worker in self.workers:
91 - executor = worker.init_cache_engine
92 - if self.use_ray:
93 - executor = executor.remote
94 - future = executor(
95 - block_size,
96 - num_gpu_blocks,
97 - num_cpu_blocks,
98 - )
99 - all_worker_futures.append(future)
100 - if self.use_ray:
101 - ray.get(all_worker_futures)
102 -
103 - def set_next(
104 - self,
105 - next_node: Union['Controller', 'Scheduler'],
106 - ) -> None:
107 - self.next_node = next_node
108 - self.is_last_stage = isinstance(next_node, Scheduler)
109 -
110 - def execute_stage(self, *args, **kwargs) -> None:
111 - all_outputs = []
112 - for worker in self.workers:
113 - executor = (worker.execute_stage.remote
114 - if self.use_ray else worker.execute_stage)
115 - output = executor(*args, **kwargs)
116 - all_outputs.append(output)
117 -
118 - if self.use_ray:
119 - all_outputs = ray.get(all_outputs)
120 -
121 - # Make sure all workers have the same results.
122 - output = all_outputs[0]
123 - for other_output in all_outputs[1:]:
124 - assert output == other_output
125 -
126 - if self.is_last_stage:
127 - self.next_node.post_step(output)
128 - else:
129 - # TODO: Support pipeline parallelism.
130 - assert False
cacheflow/worker/worker.py
+73 -93
@@ -1,14 +1,13 @@
1 """A GPU worker class."""
2 -from typing import Dict, List, Optional, Tuple
2 +from typing import Dict, List, Tuple
3
4 import torch
5
6 -from cacheflow.model_executor import (get_model, get_cache_block_size,
7 - InputMetadata, set_random_seed)
6 +from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
7 + SchedulerConfig)
8 +from cacheflow.model_executor import get_model, InputMetadata, set_random_seed
9 from cacheflow.model_executor.parallel_utils.parallel_state import (
9 - initialize_model_parallel,
10 - initialize_all_reduce_launcher,
11 - get_tensor_model_parallel_world_size)
10 + initialize_model_parallel, initialize_all_reduce_launcher)
11 from cacheflow.sampling_params import SamplingParams
12 from cacheflow.sequence import (SequenceData, SequenceGroupMetadata,
13 SequenceOutputs)
@@ -26,59 +25,46 @@ class Worker:
25
26 def __init__(
27 self,
29 - model_name: str,
30 - dtype: str,
31 - seed: int,
32 - distributed_init_method: str,
28 + model_config: ModelConfig,
29 + parallel_config: ParallelConfig,
30 + scheduler_config: SchedulerConfig,
31 rank: int,
34 - world_size: int,
35 - cache_dir: Optional[str],
36 - use_dummy_weights: bool,
37 - use_np_cache: bool,
38 - max_num_batched_tokens: int,
39 - max_num_sequences: int,
40 - tensor_parallel_size: int = 1,
41 - pipeline_parallel_size: int = 1,
32 + distributed_init_method: str,
33 ) -> None:
43 - self.init_distributed_environment(distributed_init_method,
44 - rank,
45 - world_size,
46 - tensor_parallel_size,
47 - pipeline_parallel_size)
48 - self.worker_id = rank
49 - self.seed = seed
50 - set_random_seed(self.seed)
34 + self.model_config = model_config
35 + self.parallel_config = parallel_config
36 + self.scheduler_config = scheduler_config
37 + self.rank = rank
38 + self.distributed_init_method = distributed_init_method
39 +
40 + # Initialize the distributed environment.
41 + _init_distributed_environment(parallel_config, rank,
42 + distributed_init_method)
43
44 # Initialize the model.
53 - self.model, self.dtype = get_model(
54 - model_name, dtype=dtype, cache_dir=cache_dir,
55 - use_dummy_weights=use_dummy_weights, use_np_cache=use_np_cache)
56 - tensor_model_parallel_world_size = (
57 - get_tensor_model_parallel_world_size())
58 - self.max_num_batched_tokens = max_num_batched_tokens
45 + set_random_seed(self.model_config.seed)
46 + self.model = get_model(model_config)
47 initialize_all_reduce_launcher(
60 - self.max_num_batched_tokens, self.model.config.hidden_size, self.dtype)
61 - self.max_num_sequences = max_num_sequences
62 - self.num_layers = self.model.config.num_hidden_layers
63 - assert self.model.config.num_attention_heads % tensor_model_parallel_world_size == 0
64 - self.num_heads = self.model.config.num_attention_heads // tensor_model_parallel_world_size
65 - self.head_size = self.model.config.hidden_size // (self.num_heads * tensor_model_parallel_world_size)
66 -
67 - # We reset the seed after initializing the model to ensure that
68 - # the random state is not affected by the model initialization.
69 - set_random_seed(seed)
70 -
71 - # Uninitialized cache engine. Will be initialized with
48 + self.scheduler_config.max_num_batched_tokens,
49 + self.model_config.get_hidden_size(),
50 + self.model_config.dtype,
51 + )
52 +
53 + # Uninitialized cache engine. Will be initialized by
54 # self.init_cache_engine().
55 + self.cache_config = None
56 self.block_size = None
57 self.cache_engine = None
58 self.cache_events = None
59 self.gpu_cache = None
60
61 @torch.inference_mode()
79 - def get_num_available_blocks(
80 - self, block_size: int, cpu_swap_space: int,
81 - gpu_memory_utilization: float) -> Tuple[int, int]:
62 + def profile_num_available_blocks(
63 + self,
64 + block_size: int,
65 + gpu_memory_utilization: float,
66 + cpu_swap_space: int,
67 + ) -> Tuple[int, int]:
68 # Profile the memory usage of the model and get the maximum number of
69 # cache blocks that can be allocated with the remaining free memory.
70 torch.cuda.empty_cache()
@@ -90,14 +76,15 @@ class Worker:
76 # Enable top-k sampling to reflect the accurate memory usage.
77 sampling_params = SamplingParams(top_p=0.99,
78 top_k=self.model.config.vocab_size - 1)
79 + max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
80 + max_num_seqs = self.scheduler_config.max_num_seqs
81 seqs = []
94 - for group_id in range(self.max_num_sequences):
95 - seq_len = (self.max_num_batched_tokens // self.max_num_sequences +
96 - (group_id < self.max_num_batched_tokens %
97 - self.max_num_sequences))
82 + for group_id in range(max_num_seqs):
83 + seq_len = (max_num_batched_tokens // max_num_seqs +
84 + (group_id < max_num_batched_tokens % max_num_seqs))
85 seq_data = SequenceData([0] * seq_len)
86 seq = SequenceGroupMetadata(
100 - group_id=group_id,
87 + request_id=str(group_id),
88 is_prompt=True,
89 seq_data={group_id: seq_data},
90 sampling_params=sampling_params,
@@ -105,13 +92,14 @@ class Worker:
92 )
93 seqs.append(seq)
94
108 - input_tokens, input_positions, input_metadata = self.prepare_inputs(seqs)
95 + input_tokens, input_positions, input_metadata = self._prepare_inputs(seqs)
96
97 # Execute the model.
98 + num_layers = self.model_config.get_num_layers(self.parallel_config)
99 self.model(
100 input_ids=input_tokens,
101 positions=input_positions,
114 - kv_caches=[(None, None)] * self.num_layers,
102 + kv_caches=[(None, None)] * num_layers,
103 input_metadata=input_metadata,
104 cache_events=None,
105 )
@@ -121,53 +109,27 @@ class Worker:
109 torch.cuda.synchronize()
110 peak_memory = torch.cuda.max_memory_allocated()
111 total_gpu_memory = get_gpu_memory()
124 - cache_block_size = get_cache_block_size(block_size, self.num_heads,
125 - self.head_size, self.num_layers,
126 - self.dtype)
112 + cache_block_size = CacheEngine.get_cache_block_size(
113 + block_size, self.model_config, self.parallel_config)
114 num_gpu_blocks = int((total_gpu_memory * gpu_memory_utilization
115 - peak_memory) // cache_block_size)
116 num_cpu_blocks = int(cpu_swap_space // cache_block_size)
117 torch.cuda.empty_cache()
131 - # Reset the seed to ensure that the model output is not affected by
132 - # the profiling.
133 - set_random_seed(self.seed)
118 +
119 + # Reset the seed to ensure that the random state is not affected by
120 + # the model initialization and profiling.
121 + set_random_seed(self.model_config.seed)
122 return num_gpu_blocks, num_cpu_blocks
123
136 - def init_cache_engine(self, block_size: int, num_gpu_blocks: int,
137 - num_cpu_blocks: int):
138 - self.block_size = block_size
124 + def init_cache_engine(self, cache_config: CacheConfig) -> None:
125 + self.cache_config = cache_config
126 + self.block_size = cache_config.block_size
127 self.cache_engine = CacheEngine(
140 - worker_id=self.worker_id,
141 - num_layers=self.num_layers,
142 - num_heads=self.num_heads,
143 - head_size=self.head_size,
144 - block_size=self.block_size,
145 - num_gpu_blocks=num_gpu_blocks,
146 - num_cpu_blocks=num_cpu_blocks,
147 - dtype=self.dtype,
148 - )
128 + self.cache_config, self.model_config, self.parallel_config)
129 self.cache_events = self.cache_engine.events
130 self.gpu_cache = self.cache_engine.gpu_cache
131
152 - def init_distributed_environment(self,
153 - distributed_init_method: str,
154 - rank: int,
155 - world_size: int,
156 - tensor_parallel_size: int = 1,
157 - pipeline_parallel_size: int = 1) -> None:
158 - """Initialize the distributed environment."""
159 - torch.distributed.init_process_group(
160 - backend='nccl',
161 - init_method=distributed_init_method,
162 - world_size=world_size,
163 - rank=rank,
164 - )
165 - # A small all_reduce for warmup.
166 - torch.distributed.all_reduce(torch.zeros(1).cuda())
167 - initialize_model_parallel(tensor_parallel_size,
168 - pipeline_parallel_size)
169 -
170 - def prepare_inputs(
132 + def _prepare_inputs(
133 self,
134 seq_group_metadata_list: List[SequenceGroupMetadata],
135 ) -> Tuple[torch.LongTensor, torch.LongTensor, InputMetadata]:
@@ -284,7 +246,7 @@ class Worker:
246 return tokens_tensor, positions_tensor, input_metadata
247
248 @torch.inference_mode()
287 - def execute_stage(
249 + def execute_model(
250 self,
251 seq_group_metadata_list: List[SequenceGroupMetadata],
252 blocks_to_swap_in: Dict[int, int],
@@ -316,7 +278,7 @@ class Worker:
278 return {}
279
280 # Prepare input tensors.
319 - input_tokens, input_positions, input_metadata = self.prepare_inputs(
281 + input_tokens, input_positions, input_metadata = self._prepare_inputs(
282 seq_group_metadata_list)
283
284 # Execute the model.
@@ -330,6 +292,24 @@ class Worker:
292 return output
293
294
295 +def _init_distributed_environment(
296 + parallel_config: ParallelConfig,
297 + rank: int,
298 + distributed_init_method: str,
299 +) -> None:
300 + """Initialize the distributed environment."""
301 + torch.distributed.init_process_group(
302 + backend="nccl",
303 + world_size=parallel_config.world_size,
304 + rank=rank,
305 + init_method=distributed_init_method,
306 + )
307 + # A small all_reduce for warmup.
308 + torch.distributed.all_reduce(torch.zeros(1).cuda())
309 + initialize_model_parallel(parallel_config.tensor_parallel_size,
310 + parallel_config.pipeline_parallel_size)
311 +
312 +
313 def _pad_to_alignment(x: List[int], multiple_of: int) -> List[int]:
314 return x + [0] * ((-len(x)) % multiple_of)
315
examples/simple_server.py new
+44
@@ -0,0 +1,44 @@
1 +import argparse
2 +import uuid
3 +
4 +from cacheflow import (add_server_arguments, initialize_server_from_args,
5 + SamplingParams)
6 +
7 +
8 +def main(args: argparse.Namespace):
9 + # Initialize the server.
10 + server = initialize_server_from_args(args)
11 +
12 + # Test the following prompts.
13 + test_prompts = [
14 + ("A robot may not injure a human being", SamplingParams()),
15 + ("To be or not to be,",
16 + SamplingParams(temperature=0.8, top_k=5, presence_penalty=0.2)),
17 + ("What is the meaning of life?",
18 + SamplingParams(n=2, temperature=0.8, top_p=0.95, frequency_penalty=0.1)),
19 + ("It is only with the heart that one can see rightly",
20 + SamplingParams(n=3, use_beam_search=True, temperature=0.0)),
21 + ]
22 +
23 + # Run the server.
24 + while True:
25 + # To test iteration-level scheduling, we add one request at each step.
26 + if test_prompts:
27 + prompt, sampling_params = test_prompts.pop(0)
28 + request_id = str(uuid.uuid4().hex[:8])
29 + server.add_request(request_id, prompt, sampling_params)
30 +
31 + request_outputs = server.step()
32 + for request_output in request_outputs:
33 + if request_output.done:
34 + print(request_output)
35 +
36 + if not (server.has_unfinished_requests() or test_prompts):
37 + break
38 +
39 +
40 +if __name__ == '__main__':
41 + parser = argparse.ArgumentParser(description='Simple CacheFlow server.')
42 + parser = add_server_arguments(parser)
43 + args = parser.parse_args()
44 + main(args)
simple_server.py deleted
-38
@@ -1,38 +0,0 @@
1 -import argparse
2 -
3 -from cacheflow.core.server import (
4 - add_server_arguments, process_server_arguments,
5 - init_local_server_and_frontend_with_arguments)
6 -from cacheflow.sampling_params import SamplingParams
7 -
8 -
9 -def main(args: argparse.Namespace):
10 - server, frontend = init_local_server_and_frontend_with_arguments(args)
11 - # Test the following inputs.
12 - test_inputs = [
13 - ("A robot may not injure a human being", {}), # Use default parameters.
14 - ("To be or not to be,", {"temperature": 0.8, "top_k": 5, "presence_penalty": 0.2}),
15 - ("What is the meaning of life?", {"n": 2, "temperature": 0.8, "top_p": 0.95, "frequency_penalty": 0.1}),
16 - ("It is only with the heart that one can see rightly", {"n": 3, "use_beam_search": True, "temperature": 0.0}),
17 - ]
18 - while True:
19 - if test_inputs:
20 - text, sampling_params_dict = test_inputs.pop(0)
21 - sampling_params = SamplingParams(**sampling_params_dict)
22 - sampling_params = frontend.add_eos_token(sampling_params)
23 - frontend.query(text, sampling_params)
24 - server.add_sequence_groups(frontend.get_inputs())
25 - updated_seq_groups = server.step()
26 - for seq_group in updated_seq_groups:
27 - if seq_group.is_finished():
28 - frontend.print_response(seq_group)
29 - if not (server.has_unfinished_requests() or test_inputs):
30 - break
31 -
32 -
33 -if __name__ == '__main__':
34 - parser = argparse.ArgumentParser(description='CacheFlow simple server.')
35 - parser = add_server_arguments(parser)
36 - args = parser.parse_args()
37 - args = process_server_arguments(args)
38 - main(args)