@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f756799b84
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Use runtime profiling to replace manual memory analyzers (#81)

Zhuohan Li committed May 19, 2023 at 11:35 UTC f756799b84f5558c82c7a049069f845b31573e9e
14 files changed +211 -478
cacheflow/core/server.py
+30 -22
@@ -6,15 +6,14 @@ try:
6 import ray
7 except ImportError:
8 ray = None
9 +import numpy as np
10 import torch
11
12 from cacheflow.core.scheduler import Scheduler
13 from cacheflow.frontend.simple_frontend import SimpleFrontend
14 from cacheflow.logger import init_logger
14 -from cacheflow.model_executor import get_memory_analyzer
15 from cacheflow.sampling_params import SamplingParams
16 from cacheflow.sequence import SequenceGroup
17 -from cacheflow.utils import get_gpu_memory, get_cpu_memory
17 from cacheflow.worker.controller import Controller, DeviceID
18
19 logger = init_logger(__name__)
@@ -34,14 +33,13 @@ class Server:
33 dtype: str,
34 seed: int,
35 swap_space: int,
36 + gpu_memory_utilization: float,
37 max_num_batched_tokens: int,
38 max_num_sequences: int,
39 num_nodes: int,
40 num_devices_per_node: int,
41 distributed_init_method: str,
42 all_stage_devices: List[List[DeviceID]],
43 - gpu_memory: int,
44 - cpu_memory: int,
43 use_ray: bool,
44 log_stats: bool,
45 ):
@@ -63,19 +61,6 @@ class Server:
61 assert self.world_size == 1, (
62 "Only support single GPU without Ray.")
63
66 - self.memory_analyzer = get_memory_analyzer(
67 - model_name=model,
68 - block_size=block_size,
69 - dtype=dtype,
70 - gpu_memory=gpu_memory,
71 - cpu_memory=cpu_memory,
72 - tensor_parallel_size=tensor_parallel_size,
73 - )
74 - self.num_gpu_blocks = self.memory_analyzer.get_max_num_gpu_blocks(
75 - max_num_batched_tokens=max_num_batched_tokens)
76 - self.num_cpu_blocks = self.memory_analyzer.get_max_num_cpu_blocks(
77 - swap_space_gib=swap_space)
78 -
64 # Create a controller for each pipeline stage.
65 self.controllers: List[Controller] = []
66 for i in range(pipeline_parallel_size):
@@ -87,19 +72,35 @@ class Server:
72 tensor_parallel_size=tensor_parallel_size,
73 distributed_init_method=distributed_init_method,
74 model_name=model,
90 - block_size=block_size,
91 - num_gpu_blocks=self.num_gpu_blocks,
92 - num_cpu_blocks=self.num_cpu_blocks,
75 dtype=dtype,
76 seed=seed,
77 cache_dir=cache_dir,
78 use_dummy_weights=use_dummy_weights,
79 use_np_cache=use_np_cache,
80 max_num_batched_tokens=max_num_batched_tokens,
81 + max_num_sequences=max_num_sequences,
82 use_ray=use_ray,
83 )
84 self.controllers.append(controller)
85
86 + # Initialize cache engine.
87 + all_worker_num_available_blocks = []
88 + for controller in self.controllers:
89 + all_worker_num_available_blocks.extend(
90 + controller.get_num_available_blocks(
91 + block_size, swap_space, gpu_memory_utilization)
92 + )
93 + # Since we use a shared centralized controller, we take the minimum
94 + # number of blocks across all workers to make sure all the memory
95 + # operators can be applied to all workers.
96 + self.num_gpu_blocks = np.min([b[0] for b in all_worker_num_available_blocks])
97 + self.num_cpu_blocks = np.min([b[1] for b in all_worker_num_available_blocks])
98 + logger.info(f'# GPU blocks: {self.num_gpu_blocks}, '
99 + f'# CPU blocks: {self.num_cpu_blocks}')
100 + for controller in self.controllers:
101 + controller.init_cache_engine(block_size, self.num_gpu_blocks,
102 + self.num_cpu_blocks)
103 +
104 # Create a scheduler.
105 self.scheduler = Scheduler(
106 controllers=self.controllers,
@@ -214,7 +215,11 @@ def initialize_cluster(
215 all_stage_devices)
216
217
218 +_GiB = 1 << 30
219 +
220 +
221 def add_server_arguments(parser: argparse.ArgumentParser):
222 + """Shared arguments for CacheFlow servers."""
223 # Model arguments
224 parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
225 parser.add_argument('--cache-dir', type=str, default=None,
@@ -238,6 +243,7 @@ def add_server_arguments(parser: argparse.ArgumentParser):
243 # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
244 parser.add_argument('--seed', type=int, default=0, help='random seed')
245 parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
246 + parser.add_argument('--gpu-memory-utilization', type=float, default=0.95, help='the percentage of GPU memory to be used for the model executor')
247 parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens per iteration')
248 parser.add_argument('--max-num-sequences', type=int, default=256, help='maximum number of sequences per iteration')
249 parser.add_argument('--log-stats', action='store_true', help='log system statistics')
@@ -245,8 +251,11 @@ def add_server_arguments(parser: argparse.ArgumentParser):
251
252
253 def process_server_arguments(args: argparse.Namespace):
254 + """Post process the parsed arguments."""
255 if args.pipeline_parallel_size * args.tensor_parallel_size > 1:
256 args.use_ray = True
257 + args.swap_space = args.swap_space * _GiB
258 + args.max_num_sequences = min(args.max_num_sequences, args.max_num_batched_tokens)
259 return args
260
261
@@ -274,14 +283,13 @@ def init_local_server_and_frontend_with_arguments(args: argparse.Namespace):
283 dtype=args.dtype,
284 seed=args.seed,
285 swap_space=args.swap_space,
286 + gpu_memory_utilization=args.gpu_memory_utilization,
287 max_num_batched_tokens=args.max_num_batched_tokens,
288 max_num_sequences=args.max_num_sequences,
289 num_nodes=num_nodes,
290 num_devices_per_node=num_devices_per_node,
291 distributed_init_method=distributed_init_method,
292 all_stage_devices=all_stage_devices,
283 - gpu_memory=get_gpu_memory(),
284 - cpu_memory=get_cpu_memory(),
293 use_ray=args.use_ray,
294 log_stats=args.log_stats,
295 )
cacheflow/frontend/fastapi_frontend.py
+7 -3
@@ -15,7 +15,7 @@ from cacheflow.core.server import (Server, add_server_arguments,
15 from cacheflow.frontend.utils import get_tokenizer
16 from cacheflow.sampling_params import SamplingParams
17 from cacheflow.sequence import Sequence, SequenceGroup
18 -from cacheflow.utils import Counter, get_gpu_memory, get_cpu_memory
18 +from cacheflow.utils import Counter
19 from cacheflow.worker.controller import DeviceID
20
21 TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
@@ -34,6 +34,7 @@ class FastAPIServer:
34 dtype: str,
35 seed: int,
36 swap_space: int,
37 + gpu_memory_utilization: float,
38 max_num_batched_tokens: int,
39 max_num_sequences: int,
40 num_nodes: int,
@@ -41,6 +42,7 @@ class FastAPIServer:
42 distributed_init_method: str,
43 all_stage_devices: List[List[DeviceID]],
44 server_use_ray: bool,
45 + log_stats: bool,
46 ):
47 self.block_size = block_size
48
@@ -62,15 +64,15 @@ class FastAPIServer:
64 dtype=dtype,
65 seed=seed,
66 swap_space=swap_space,
67 + gpu_memory_utilization=gpu_memory_utilization,
68 max_num_batched_tokens=max_num_batched_tokens,
69 max_num_sequences=max_num_sequences,
70 num_nodes=num_nodes,
71 num_devices_per_node=num_devices_per_node,
72 distributed_init_method=distributed_init_method,
73 all_stage_devices=all_stage_devices,
71 - gpu_memory=get_gpu_memory(),
72 - cpu_memory=get_cpu_memory(),
74 use_ray=server_use_ray,
75 + log_stats=log_stats,
76 )
77
78 self.running_seq_groups: Dict[int, SequenceGroup] = {}
@@ -182,6 +184,7 @@ if __name__ == "__main__":
184 dtype=args.dtype,
185 seed=args.seed,
186 swap_space=args.swap_space,
187 + gpu_memory_utilization=args.gpu_memory_utilization,
188 max_num_batched_tokens=args.max_num_batched_tokens,
189 max_num_sequences=args.max_num_sequences,
190 num_nodes=num_nodes,
@@ -189,6 +192,7 @@ if __name__ == "__main__":
192 distributed_init_method=distributed_init_method,
193 all_stage_devices=all_stage_devices,
194 server_use_ray=args.use_ray,
195 + log_stats=args.log_stats,
196 )
197
198 uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/model_executor/__init__.py
+4 -3
@@ -1,11 +1,12 @@
1 from cacheflow.model_executor.input_metadata import InputMetadata
2 -from cacheflow.model_executor.model_loader import get_model, get_memory_analyzer
3 -from cacheflow.model_executor.utils import set_random_seed
2 +from cacheflow.model_executor.model_loader import get_model
3 +from cacheflow.model_executor.utils import (set_random_seed,
4 + get_cache_block_size)
5
6
7 __all__ = [
8 "InputMetadata",
9 + "get_cache_block_size",
10 "get_model",
9 - "get_memory_analyzer",
11 "set_random_seed",
12 ]
cacheflow/model_executor/layers/attention.py
+29 -21
@@ -11,6 +11,8 @@ from cacheflow import pos_encoding_ops
11 from cacheflow.model_executor.input_metadata import InputMetadata
12
13
14 +_SUPPORTED_HEAD_SIZES = [32, 64, 80, 96, 128, 160, 192, 256]
15 +
16 class GPTCacheFlowAttention(nn.Module):
17 """GPT-style multi-head attention.
18
@@ -39,11 +41,19 @@ class GPTCacheFlowAttention(nn.Module):
41 5. Output a flattened 1D tensor.
42 """
43
42 - def __init__(self, scale: float) -> None:
44 + def __init__(self, num_heads: int, head_size: int, scale: float) -> None:
45 super().__init__()
46 + self.num_heads = num_heads
47 + self.head_size = head_size
48 self.scale = float(scale)
49 self.attn_op = xops.fmha.cutlass.FwOp()
50
51 + if self.head_size not in _SUPPORTED_HEAD_SIZES:
52 + raise ValueError(f'head_size ({self.head_size}) is not supported by '
53 + 'the single_query_cached_kv_attention kernel. '
54 + 'Use one of the following head sizes: '
55 + f'{_SUPPORTED_HEAD_SIZES}.')
56 +
57 def multi_query_kv_attention(
58 self,
59 output: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
@@ -74,14 +84,6 @@ class GPTCacheFlowAttention(nn.Module):
84 value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
85 input_metadata: InputMetadata,
86 ) -> None:
77 - head_size = value_cache.shape[2]
78 - supported_head_sizes = [32, 64, 80, 96, 128, 160, 192, 256]
79 - if head_size not in supported_head_sizes:
80 - raise ValueError(f'head_size ({head_size}) is not supported by '
81 - 'the single_query_cached_kv_attention kernel. '
82 - 'Use one of the following head sizes: '
83 - f'{supported_head_sizes}.')
84 -
87 block_size = value_cache.shape[3]
88 attention_ops.single_query_cached_kv_attention(
89 output,
@@ -100,8 +102,8 @@ class GPTCacheFlowAttention(nn.Module):
102 query: torch.Tensor, # [num_tokens, num_heads * head_size]
103 key: torch.Tensor, # [num_tokens, num_heads * head_size]
104 value: torch.Tensor, # [num_tokens, num_heads * head_size]
103 - key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
104 - value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
105 + key_cache: Optional[torch.Tensor], # [num_blocks, num_heads, head_size/x, block_size, x]
106 + value_cache: Optional[torch.Tensor], # [num_blocks, num_heads, head_size, block_size]
107 input_metadata: InputMetadata,
108 cache_event: Optional[torch.cuda.Event],
109 ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
@@ -109,11 +111,9 @@ class GPTCacheFlowAttention(nn.Module):
111 # tensor of shape [num_tokens, 3 * num_heads * head_size].
112
113 # Reshape the query, key, and value tensors.
112 - num_heads = value_cache.shape[1]
113 - head_size = value_cache.shape[2]
114 - query = query.view(-1, num_heads, head_size)
115 - key = key.view(-1, num_heads, head_size)
116 - value = value.view(-1, num_heads, head_size)
114 + query = query.view(-1, self.num_heads, self.head_size)
115 + key = key.view(-1, self.num_heads, self.head_size)
116 + value = value.view(-1, self.num_heads, self.head_size)
117
118 # Pre-allocate the output tensor.
119 output = torch.empty_like(query)
@@ -134,8 +134,11 @@ class GPTCacheFlowAttention(nn.Module):
134 cache_event.wait()
135
136 # Reshape the keys and values and store them in the cache.
137 + # When key_cache and value_cache are not provided, the new key
138 + # and value vectors will not be cached.
139 num_valid_tokens = input_metadata.num_valid_tokens
138 - if num_valid_tokens > 0:
140 + if (num_valid_tokens > 0 and key_cache is not None
141 + and value_cache is not None):
142 # The stride is 3 because the key and value are sliced from qkv.
143 cache_ops.reshape_and_cache(
144 key[:num_valid_tokens],
@@ -146,6 +149,10 @@ class GPTCacheFlowAttention(nn.Module):
149 )
150
151 if input_metadata.num_generation_tokens > 0:
152 + assert key_cache is not None and value_cache is not None, (
153 + "key_cache and value_cache must be provided when "
154 + "generating tokens."
155 + )
156 # Compute the attention op for generation tokens.
157 self.single_query_cached_kv_attention(
158 output[num_prompt_tokens:num_valid_tokens],
@@ -156,7 +163,7 @@ class GPTCacheFlowAttention(nn.Module):
163
164 # Reshape the output tensor.
165 # NOTE(woosuk): The output tensor may include paddings.
159 - return output.view(-1, num_heads * head_size)
166 + return output.view(-1, self.num_heads * self.head_size)
167
168
169 class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
@@ -164,12 +171,14 @@ class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
171
172 def __init__(
173 self,
174 + num_heads: int,
175 + head_size: int,
176 scale: float,
177 rotary_dim: int,
178 max_position: int = 8192,
179 base: int = 10000,
180 ) -> None:
172 - super().__init__(scale)
181 + super().__init__(num_heads, head_size, scale)
182
183 # Create the cos and sin cache.
184 inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2) / rotary_dim))
@@ -199,12 +208,11 @@ class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
208 ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
209 # Apply rotary embedding to the query and key before passing them
210 # to the attention op.
202 - head_size = value_cache.shape[2]
211 pos_encoding_ops.rotary_embedding_neox(
212 positions,
213 query,
214 key,
207 - head_size,
215 + self.head_size,
216 self.cos_sin_cache,
217 )
218 return super().forward(
cacheflow/model_executor/layers/sampler.py
+1 -1
@@ -74,7 +74,7 @@ class Sampler(nn.Module):
74 # Apply top-p and top-k truncation.
75 top_ps, top_ks = _get_top_p_top_k(input_metadata, self.vocab_size)
76 assert len(top_ps) == len(top_ks) == probs.shape[0]
77 - if any(p < 1.0 for p in top_ps) or any(k != -1 for k in top_ks):
77 + if any(p < 1.0 for p in top_ps) or any(k != self.vocab_size for k in top_ks):
78 probs = _apply_top_p_top_k(probs, top_ps, top_ks)
79
80 # Sample the next tokens.
cacheflow/model_executor/memory_analyzer.py deleted
-370
@@ -1,370 +0,0 @@
1 -import torch
2 -from transformers import AutoConfig
3 -
4 -from cacheflow.logger import init_logger
5 -from cacheflow.model_executor.utils import get_dtype_size
6 -
7 -logger = init_logger(__name__)
8 -
9 -_GiB = 1 << 30
10 -
11 -
12 -class CacheFlowMemoryAnalyzer:
13 -
14 - def get_max_num_gpu_blocks(
15 - self,
16 - max_num_batched_tokens: int,
17 - memory_utilization: float,
18 - ) -> int:
19 - raise NotImplementedError()
20 -
21 - def get_workspace_size(self) -> int:
22 - return 1 * _GiB
23 -
24 - def get_cache_block_size(self) -> int:
25 - raise NotImplementedError()
26 -
27 - def get_max_num_cpu_blocks(
28 - self,
29 - swap_space_gib: int,
30 - ) -> int:
31 - swap_space = swap_space_gib * _GiB
32 - cpu_memory = self.cpu_memory
33 - if swap_space > 0.8 * cpu_memory:
34 - raise ValueError(f'The swap space ({swap_space_gib:.2f} GiB) '
35 - 'takes more than 80% of the available memory '
36 - f'({cpu_memory / _GiB:.2f} GiB).'
37 - 'Please check the swap space size.')
38 - if swap_space > 0.5 * cpu_memory:
39 - logger.info(f'WARNING: The swap space ({swap_space_gib:.2f} GiB) '
40 - 'takes more than 50% of the available memory '
41 - f'({cpu_memory / _GiB:.2f} GiB).'
42 - 'This may slow the system performance.')
43 - max_num_blocks = swap_space // self.get_cache_block_size()
44 - return max_num_blocks
45 -
46 - def get_param_size(self) -> int:
47 - raise NotImplementedError()
48 -
49 - def get_max_act_size(self, max_num_batched_tokens: int) -> int:
50 - raise NotImplementedError()
51 -
52 - def get_cache_block_size(self) -> int:
53 - key_cache_block = self.block_size * self.hidden_size // self.tensor_parallel_size
54 - value_cache_block = key_cache_block
55 - total = self.num_layers * (key_cache_block + value_cache_block)
56 - dtype_size = get_dtype_size(self.dtype)
57 - return dtype_size * total
58 -
59 - def get_max_num_gpu_blocks(
60 - self,
61 - max_num_batched_tokens: int,
62 - memory_utilization: float = 0.95,
63 - ) -> int:
64 - # NOTE(woosuk): This assumes that the machine has homogeneous GPUs.
65 - usable_memory = int(memory_utilization * self.gpu_memory)
66 -
67 - param_size = self.get_param_size()
68 - act_size = self.get_max_act_size(max_num_batched_tokens)
69 - workspace_size = self.get_workspace_size()
70 -
71 - max_cache_size = usable_memory - (param_size + act_size + workspace_size)
72 - if max_cache_size <= 0:
73 - raise RuntimeError('Not enough GPU memory.')
74 - max_num_blocks = max_cache_size // self.get_cache_block_size()
75 - return max_num_blocks
76 -
77 -
78 -class GPT2MemoryAnalyzer(CacheFlowMemoryAnalyzer):
79 -
80 - def __init__(
81 - self,
82 - model_name: str,
83 - block_size: int,
84 - dtype: torch.dtype,
85 - gpu_memory: int,
86 - cpu_memory: int,
87 - tensor_parallel_size: int,
88 - ) -> None:
89 - self.model_name = model_name
90 - self.block_size = block_size
91 - self.dtype = dtype
92 - self.gpu_memory = gpu_memory
93 - self.cpu_memory = cpu_memory
94 - self.tensor_parallel_size = tensor_parallel_size
95 -
96 - config = AutoConfig.from_pretrained(model_name)
97 - self.num_layers = config.num_hidden_layers
98 - self.hidden_size = config.hidden_size
99 - self.num_heads = config.num_attention_heads
100 - self.head_size = config.hidden_size // self.num_heads
101 - self.ffn_size = config.n_inner if config.n_inner is not None else 4 * self.hidden_size
102 - self.vocab_size = config.vocab_size
103 - self.max_position = config.max_position_embeddings
104 -
105 - def get_param_size(self) -> int:
106 - word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
107 - position_embedding = self.max_position * self.hidden_size
108 -
109 - ln1 = 2 * self.hidden_size
110 - q = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
111 - k = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
112 - v = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
113 - out = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
114 - mha = ln1 + q + k + v + out
115 -
116 - ln2 = 2 * self.hidden_size
117 - ffn1 = self.hidden_size * self.ffn_size // self.tensor_parallel_size + self.ffn_size
118 - ffn2 = self.ffn_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
119 - ffn = ln2 + ffn1 + ffn2
120 -
121 - total = (word_embedding + position_embedding +
122 - self.num_layers * (mha + ffn))
123 - dtype_size = get_dtype_size(self.dtype)
124 - return dtype_size * total
125 -
126 - def get_max_act_size(
127 - self,
128 - max_num_batched_tokens: int,
129 - ) -> int:
130 - # NOTE: We approxmiately calculate the maximum activation size by
131 - # estimating
132 - # 1) the maximum activation tensor size during inference
133 - # 2) the residual tensor size during inference
134 - # Here, we assume that FlashAttention is used and
135 - # thus the attention maps are never materialized in GPU DRAM.
136 - residual = max_num_batched_tokens * self.hidden_size
137 - qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
138 - ffn = max_num_batched_tokens * self.ffn_size // self.tensor_parallel_size
139 - # Double the activation size for input and output.
140 - max_act = 2 * (max(qkv, ffn) + residual)
141 - # Size of output logits.
142 - output_logits = 2 * (max_num_batched_tokens * self.vocab_size)
143 - max_act = max(max_act, output_logits)
144 - dtype_size = get_dtype_size(self.dtype)
145 - return dtype_size * max_act
146 -
147 -
148 -class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
149 -
150 - def __init__(
151 - self,
152 - model_name: str,
153 - block_size: int,
154 - dtype: torch.dtype,
155 - gpu_memory: int,
156 - cpu_memory: int,
157 - tensor_parallel_size: int,
158 - ) -> None:
159 - self.model_name = model_name
160 - self.block_size = block_size
161 - self.dtype = dtype
162 - self.gpu_memory = gpu_memory
163 - self.cpu_memory = cpu_memory
164 - self.tensor_parallel_size = tensor_parallel_size
165 -
166 - config = AutoConfig.from_pretrained(model_name)
167 - self.num_layers = config.num_hidden_layers
168 - self.hidden_size = config.hidden_size
169 - self.num_heads = config.num_attention_heads
170 - self.head_size = config.hidden_size // self.num_heads
171 - self.ffn_size = config.ffn_dim
172 - self.embedding_size = config.word_embed_proj_dim
173 - self.vocab_size = config.vocab_size
174 - self.max_position = config.max_position_embeddings
175 -
176 - def get_param_size(self) -> int:
177 - word_embedding = self.vocab_size * self.embedding_size // self.tensor_parallel_size
178 - if self.embedding_size != self.hidden_size:
179 - # Project in/out.
180 - word_embedding += 2 * self.embedding_size * self.hidden_size
181 - position_embedding = self.max_position * self.hidden_size
182 -
183 - ln1 = 2 * self.hidden_size
184 - q = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
185 - k = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
186 - v = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
187 - out = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
188 - mha = ln1 + q + k + v + out
189 -
190 - ln2 = 2 * self.hidden_size
191 - ffn1 = self.hidden_size * self.ffn_size // self.tensor_parallel_size + self.ffn_size
192 - ffn2 = self.ffn_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
193 - ffn = ln2 + ffn1 + ffn2
194 -
195 - total = (word_embedding + position_embedding +
196 - self.num_layers * (mha + ffn))
197 - dtype_size = get_dtype_size(self.dtype)
198 - return dtype_size * total
199 -
200 - def get_max_act_size(
201 - self,
202 - max_num_batched_tokens: int,
203 - ) -> int:
204 - # NOTE: We approxmiately calculate the maximum activation size by
205 - # estimating
206 - # 1) the maximum activation tensor size during inference
207 - # 2) the residual tensor size during inference
208 - # Here, we assume that we use memory-efficient attention which
209 - # does not materialize the attention maps in GPU DRAM.
210 - residual = max_num_batched_tokens * self.hidden_size
211 - qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
212 - ffn = max_num_batched_tokens * self.ffn_size // self.tensor_parallel_size
213 - # Double the activation size for input and output.
214 - max_act = 2 * (max(qkv, ffn) + residual)
215 - # Size of output logits.
216 - output_logits = 2 * (max_num_batched_tokens * self.vocab_size)
217 - max_act = max(max_act, output_logits)
218 - dtype_size = get_dtype_size(self.dtype)
219 - return dtype_size * max_act
220 -
221 -
222 -class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
223 -
224 - def __init__(
225 - self,
226 - model_name: str,
227 - block_size: int,
228 - dtype: torch.dtype,
229 - gpu_memory: int,
230 - cpu_memory: int,
231 - tensor_parallel_size: int,
232 - ) -> None:
233 - self.model_name = model_name
234 - self.block_size = block_size
235 - self.dtype = dtype
236 - self.gpu_memory = gpu_memory
237 - self.cpu_memory = cpu_memory
238 - self.tensor_parallel_size = tensor_parallel_size
239 -
240 - config = AutoConfig.from_pretrained(model_name)
241 - self.num_layers = config.num_hidden_layers
242 - self.hidden_size = config.hidden_size
243 - self.num_heads = config.num_attention_heads
244 - self.head_size = config.hidden_size // self.num_heads
245 - self.ffn_size = config.intermediate_size
246 - self.vocab_size = config.vocab_size
247 - self.max_position = 8192
248 -
249 - def get_param_size(self) -> int:
250 - # NOTE: LLaMA does not tie the two embeddings.
251 - word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
252 - lm_head = self.vocab_size * self.hidden_size // self.tensor_parallel_size
253 -
254 - # NOTE: LLaMA does not have bias terms.
255 - ln1 = self.hidden_size
256 - q = self.hidden_size * self.hidden_size // self.tensor_parallel_size
257 - k = self.hidden_size * self.hidden_size // self.tensor_parallel_size
258 - v = self.hidden_size * self.hidden_size // self.tensor_parallel_size
259 - out = self.hidden_size * self.hidden_size // self.tensor_parallel_size
260 - # Rotary embedding.
261 - # TODO(woosuk): Share the rotary embedding between layers.
262 - rot = self.max_position * self.head_size
263 - mha = ln1 + q + k + v + out + rot
264 -
265 - ln2 = self.hidden_size
266 - gate = self.hidden_size * self.ffn_size // self.tensor_parallel_size
267 - down = self.ffn_size * self.hidden_size // self.tensor_parallel_size
268 - up = self.hidden_size * self.ffn_size // self.tensor_parallel_size
269 - ffn = ln2 + gate + down + up
270 -
271 - total = word_embedding + self.num_layers * (mha + ffn) + lm_head
272 - dtype_size = get_dtype_size(self.dtype)
273 - return dtype_size * total
274 -
275 - def get_max_act_size(
276 - self,
277 - max_num_batched_tokens: int,
278 - ) -> int:
279 - # NOTE: We approxmiately calculate the maximum activation size by
280 - # estimating
281 - # 1) the maximum activation tensor size during inference
282 - # 2) the residual tensor size during inference
283 - # Here, we assume that we use memory-efficient attention which
284 - # does not materialize the attention maps in GPU DRAM.
285 - residual = max_num_batched_tokens * self.hidden_size
286 - qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
287 - ffn = 2 * (max_num_batched_tokens * self.ffn_size) // self.tensor_parallel_size
288 - # Double the activation size for input and output.
289 - max_act = 2 * (max(qkv, ffn) + residual)
290 - # Size of output logits.
291 - output_logits = 2 * (max_num_batched_tokens * self.vocab_size)
292 - max_act = max(max_act, output_logits)
293 - dtype_size = get_dtype_size(self.dtype)
294 - return dtype_size * max_act
295 -
296 -
297 -class GPTNeoXMemoryAnalyzer(CacheFlowMemoryAnalyzer):
298 -
299 - def __init__(
300 - self,
301 - model_name: str,
302 - block_size: int,
303 - dtype: torch.dtype,
304 - gpu_memory: int,
305 - cpu_memory: int,
306 - tensor_parallel_size: int,
307 - ) -> None:
308 - self.model_name = model_name
309 - self.block_size = block_size
310 - self.dtype = dtype
311 - self.gpu_memory = gpu_memory
312 - self.cpu_memory = cpu_memory
313 - self.tensor_parallel_size = tensor_parallel_size
314 -
315 - config = AutoConfig.from_pretrained(model_name)
316 - self.num_layers = config.num_hidden_layers
317 - self.hidden_size = config.hidden_size
318 - self.num_heads = config.num_attention_heads
319 - self.head_size = config.hidden_size // self.num_heads
320 - self.ffn_size = config.intermediate_size
321 - self.vocab_size = config.vocab_size
322 - self.max_position = 8192
323 - self.tie_word_embeddings = config.tie_word_embeddings
324 -
325 - def get_param_size(self) -> int:
326 - word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
327 - if self.tie_word_embeddings:
328 - lm_head = 0
329 - else:
330 - lm_head = self.vocab_size * self.hidden_size // self.tensor_parallel_size
331 -
332 - ln1 = 2 * self.hidden_size
333 - q = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
334 - k = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
335 - v = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
336 - out = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
337 - # Rotary embedding.
338 - # TODO(woosuk): Share the rotary embedding between layers.
339 - rot = self.max_position * self.head_size
340 - mha = ln1 + q + k + v + out + rot
341 -
342 - ln2 = 2 * self.hidden_size
343 - ffn1 = self.hidden_size * self.ffn_size // self.tensor_parallel_size + self.ffn_size
344 - ffn2 = self.ffn_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
345 - ffn = ln2 + ffn1 + ffn2
346 -
347 - total = word_embedding + self.num_layers * (mha + ffn) + lm_head
348 - dtype_size = get_dtype_size(self.dtype)
349 - return dtype_size * total
350 -
351 - def get_max_act_size(
352 - self,
353 - max_num_batched_tokens: int,
354 - ) -> int:
355 - # NOTE: We approxmiately calculate the maximum activation size by
356 - # estimating
357 - # 1) the maximum activation tensor size during inference
358 - # 2) the residual tensor size during inference
359 - # Here, we assume that we use memory-efficient attention which
360 - # does not materialize the attention maps in GPU DRAM.
361 - residual = max_num_batched_tokens * self.hidden_size
362 - qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
363 - ffn = 2 * (max_num_batched_tokens * self.ffn_size) // self.tensor_parallel_size
364 - # Double the activation size for input and output.
365 - max_act = 2 * (max(qkv, ffn) + residual)
366 - # Size of output logits.
367 - output_logits = 2 * (max_num_batched_tokens * self.vocab_size)
368 - max_act = max(max_act, output_logits)
369 - dtype_size = get_dtype_size(self.dtype)
370 - return dtype_size * max_act
cacheflow/model_executor/model_loader.py
-37
@@ -5,9 +5,6 @@ import torch
5 import torch.nn as nn
6 from transformers import AutoConfig, PretrainedConfig
7
8 -from cacheflow.model_executor.memory_analyzer import (
9 - CacheFlowMemoryAnalyzer, GPT2MemoryAnalyzer, GPTNeoXMemoryAnalyzer,
10 - LlamaMemoryAnalyzer, OPTMemoryAnalyzer)
8 from cacheflow.model_executor.models import (
9 GPT2LMHeadModel, GPTNeoXForCausalLM, LlamaForCausalLM, OPTForCausalLM)
10 from cacheflow.model_executor.utils import get_torch_dtype
@@ -22,14 +19,6 @@ _MODEL_REGISTRY = {
19 "OPTForCausalLM": OPTForCausalLM,
20 }
21
25 -_MEMORY_ANALYZERS = {
26 - "GPT2LMHeadModel": GPT2MemoryAnalyzer,
27 - "GPTNeoXForCausalLM": GPTNeoXMemoryAnalyzer,
28 - "LlamaForCausalLM": LlamaMemoryAnalyzer,
29 - "OPTForCausalLM": OPTMemoryAnalyzer,
30 -}
31 -
32 -
22 def _get_model_architecture(config: PretrainedConfig) -> nn.Module:
23 architectures = getattr(config, "architectures", [])
24 for arch in architectures:
@@ -41,17 +30,6 @@ def _get_model_architecture(config: PretrainedConfig) -> nn.Module:
30 )
31
32
44 -def _get_memory_analyzer(config: PretrainedConfig) -> CacheFlowMemoryAnalyzer:
45 - architectures = getattr(config, "architectures", [])
46 - for arch in architectures:
47 - if arch in _MEMORY_ANALYZERS:
48 - return _MEMORY_ANALYZERS[arch]
49 - raise ValueError(
50 - f"Model architectures {architectures} are not supported for now. "
51 - f"Supported architectures: {list(_MEMORY_ANALYZERS.keys())}"
52 - )
53 -
54 -
33 def _get_dtype(config: PretrainedConfig, dtype: str) -> torch.dtype:
34 # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
35 # because config.torch_dtype can be None.
@@ -100,18 +78,3 @@ def get_model(
78 model = model.cuda()
79 return model.eval(), torch_dtype
80
103 -
104 -def get_memory_analyzer(
105 - model_name: str,
106 - block_size: int,
107 - dtype: str,
108 - gpu_memory: int,
109 - cpu_memory: int,
110 - tensor_parallel_size: int = 1,
111 -) -> CacheFlowMemoryAnalyzer:
112 - config = AutoConfig.from_pretrained(model_name)
113 - torch_dtype = _get_dtype(config, dtype)
114 - memory_analyzer = _get_memory_analyzer(config)
115 - return memory_analyzer(
116 - model_name, block_size, torch_dtype, gpu_memory, cpu_memory,
117 - tensor_parallel_size)
cacheflow/model_executor/models/gpt2.py
+2 -1
@@ -58,7 +58,8 @@ class GPT2Attention(nn.Module):
58 self.c_proj = RowParallelLinear(self.hidden_size, self.hidden_size, bias=True,
59 input_is_parallel=True,
60 perform_initialization=False)
61 - self.attn = GPTCacheFlowAttention(scale=self.scale)
61 + self.attn = GPTCacheFlowAttention(self.num_heads, self.head_dim,
62 + scale=self.scale)
63
64 def forward(
65 self,
cacheflow/model_executor/models/gpt_neox.py
+2 -1
@@ -62,7 +62,8 @@ class GPTNeoXAttention(nn.Module):
62 scaling = self.head_size ** -0.5
63 rotary_dim = int(self.head_size * config.rotary_pct)
64 assert rotary_dim % 2 == 0
65 - self.attn = GPTNeoXCacheFlowAttention(scaling, rotary_dim)
65 + self.attn = GPTNeoXCacheFlowAttention(self.num_heads, self.head_size,
66 + scaling, rotary_dim)
67
68 def forward(
69 self,
cacheflow/model_executor/models/llama.py
+2 -1
@@ -104,7 +104,8 @@ class LlamaAttention(nn.Module):
104 input_is_parallel=True,
105 perform_initialization=False,
106 )
107 - self.attn = GPTNeoXCacheFlowAttention(self.scaling, self.head_dim)
107 + self.attn = GPTNeoXCacheFlowAttention(self.num_heads, self.head_dim,
108 + self.scaling, rotary_dim=self.head_dim)
109
110 def forward(
111 self,
cacheflow/model_executor/models/opt.py
+2 -1
@@ -74,7 +74,8 @@ class OPTAttention(nn.Module):
74 self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
75 input_is_parallel=True,
76 perform_initialization=False)
77 - self.attn = GPTCacheFlowAttention(scale=self.scaling)
77 + self.attn = GPTCacheFlowAttention(self.num_heads, self.head_dim,
78 + scale=self.scaling)
79
80 def forward(
81 self,
cacheflow/model_executor/utils.py
+12
@@ -40,3 +40,15 @@ def set_random_seed(seed: int) -> None:
40
41 if model_parallel_is_initialized():
42 model_parallel_cuda_manual_seed(seed)
43 +
44 +
45 +def get_cache_block_size(block_size: int,
46 + num_heads: int,
47 + head_size: int,
48 + num_layers: int,
49 + dtype: str) -> int:
50 + key_cache_block = block_size * num_heads * head_size
51 + value_cache_block = key_cache_block
52 + total = num_layers * (key_cache_block + value_cache_block)
53 + dtype_size = get_dtype_size(dtype)
54 + return dtype_size * total
cacheflow/worker/controller.py
+36 -9
@@ -23,23 +23,18 @@ class Controller:
23 pipeline_parallel_size: int,
24 distributed_init_method: str,
25 model_name: str,
26 - block_size: int,
27 - num_gpu_blocks: int,
28 - num_cpu_blocks: int,
26 dtype: str,
27 seed: int,
28 cache_dir: Optional[str],
29 use_dummy_weights: bool,
30 use_np_cache: bool,
31 max_num_batched_tokens: int,
32 + max_num_sequences: int,
33 use_ray: bool,
34 ) -> None:
35 self.stage_id = stage_id
36 self.stage_devices = stage_devices
37 self.model_name = model_name
40 - self.block_size = block_size
41 - self.num_gpu_blocks = num_gpu_blocks
42 - self.num_cpu_blocks = num_cpu_blocks
38 self.use_ray = use_ray
39
40 # Which pipeline stage is this node assigned to?
@@ -56,9 +51,6 @@ class Controller:
51 worker_cls = Worker
52 worker = worker_cls(
53 model_name=model_name,
59 - block_size=block_size,
60 - num_gpu_blocks=num_gpu_blocks,
61 - num_cpu_blocks=num_cpu_blocks,
54 dtype=dtype,
55 seed=seed,
56 distributed_init_method=distributed_init_method,
@@ -70,9 +62,44 @@ class Controller:
62 use_dummy_weights=use_dummy_weights,
63 use_np_cache=use_np_cache,
64 max_num_batched_tokens=max_num_batched_tokens,
65 + max_num_sequences=max_num_sequences,
66 )
67 self.workers.append(worker)
68
69 + def get_num_available_blocks(self, block_size: int, cpu_swap_space: int,
70 + gpu_memory_utilization: float) -> List[Tuple[int, int]]:
71 + all_worker_results = []
72 + for worker in self.workers:
73 + executor = worker.get_num_available_blocks
74 + if self.use_ray:
75 + executor = executor.remote
76 +
77 + result = executor(
78 + block_size,
79 + cpu_swap_space,
80 + gpu_memory_utilization,
81 + )
82 + all_worker_results.append(result)
83 + if self.use_ray:
84 + all_worker_results = ray.get(all_worker_results)
85 + return all_worker_results
86 +
87 + def init_cache_engine(self, block_size: int, num_gpu_blocks: int,
88 + num_cpu_blocks: int):
89 + all_worker_futures = []
90 + for worker in self.workers:
91 + executor = worker.init_cache_engine
92 + if self.use_ray:
93 + executor = executor.remote
94 + future = executor(
95 + block_size,
96 + num_gpu_blocks,
97 + num_cpu_blocks,
98 + )
99 + all_worker_futures.append(future)
100 + if self.use_ray:
101 + ray.get(all_worker_futures)
102 +
103 def set_next(
104 self,
105 next_node: Union['Controller', 'Scheduler'],
cacheflow/worker/worker.py
+84 -8
@@ -3,7 +3,8 @@ from typing import Dict, List, Optional, Tuple
3
4 import torch
5
6 -from cacheflow.model_executor import get_model, InputMetadata, set_random_seed
6 +from cacheflow.model_executor import (get_model, get_cache_block_size,
7 + InputMetadata, set_random_seed)
8 from cacheflow.model_executor.parallel_utils.parallel_state import (
9 initialize_model_parallel,
10 initialize_all_reduce_launcher,
@@ -12,6 +13,7 @@ from cacheflow.sampling_params import SamplingParams
13 from cacheflow.sequence import (SequenceData, SequenceGroupMetadata,
14 SequenceOutputs)
15 from cacheflow.worker.cache_engine import CacheEngine
16 +from cacheflow.utils import get_gpu_memory
17
18
19 class Worker:
@@ -25,9 +27,6 @@ class Worker:
27 def __init__(
28 self,
29 model_name: str,
28 - block_size: int,
29 - num_gpu_blocks: int,
30 - num_cpu_blocks: int,
30 dtype: str,
31 seed: int,
32 distributed_init_method: str,
@@ -37,6 +36,7 @@ class Worker:
36 use_dummy_weights: bool,
37 use_np_cache: bool,
38 max_num_batched_tokens: int,
39 + max_num_sequences: int,
40 tensor_parallel_size: int = 1,
41 pipeline_parallel_size: int = 1,
42 ) -> None:
@@ -46,8 +46,8 @@ class Worker:
46 tensor_parallel_size,
47 pipeline_parallel_size)
48 self.worker_id = rank
49 - self.block_size = block_size
50 - set_random_seed(seed)
49 + self.seed = seed
50 + set_random_seed(self.seed)
51
52 # Initialize the model.
53 self.model, self.dtype = get_model(
@@ -55,8 +55,10 @@ class Worker:
55 use_dummy_weights=use_dummy_weights, use_np_cache=use_np_cache)
56 tensor_model_parallel_world_size = (
57 get_tensor_model_parallel_world_size())
58 + self.max_num_batched_tokens = max_num_batched_tokens
59 initialize_all_reduce_launcher(
59 - max_num_batched_tokens, self.model.config.hidden_size, self.dtype)
60 + self.max_num_batched_tokens, self.model.config.hidden_size, self.dtype)
61 + self.max_num_sequences = max_num_sequences
62 self.num_layers = self.model.config.num_hidden_layers
63 assert self.model.config.num_attention_heads % tensor_model_parallel_world_size == 0
64 self.num_heads = self.model.config.num_attention_heads // tensor_model_parallel_world_size
@@ -66,12 +68,80 @@ class Worker:
68 # the random state is not affected by the model initialization.
69 set_random_seed(seed)
70
71 + # Uninitialized cache engine. Will be initialized with
72 + # self.init_cache_engine().
73 + self.block_size = None
74 + self.cache_engine = None
75 + self.cache_events = None
76 + self.gpu_cache = None
77 +
78 + @torch.inference_mode()
79 + def get_num_available_blocks(
80 + self, block_size: int, cpu_swap_space: int,
81 + gpu_memory_utilization: float) -> Tuple[int, int]:
82 + # Profile the memory usage of the model and get the maximum number of
83 + # cache blocks that can be allocated with the remaining free memory.
84 + torch.cuda.empty_cache()
85 + torch.cuda.reset_peak_memory_stats()
86 +
87 + # Profile memory usage with max_num_sequences sequences and the total
88 + # number of tokens equal to max_num_batched_tokens.
89 +
90 + # Enable top-k sampling to reflect the accurate memory usage.
91 + sampling_params = SamplingParams(top_p=0.99,
92 + top_k=self.model.config.vocab_size - 1)
93 + seqs = []
94 + for group_id in range(self.max_num_sequences):
95 + seq_len = (self.max_num_batched_tokens // self.max_num_sequences +
96 + (group_id < self.max_num_batched_tokens %
97 + self.max_num_sequences))
98 + seq_data = SequenceData([0] * seq_len)
99 + seq = SequenceGroupMetadata(
100 + group_id=group_id,
101 + is_prompt=True,
102 + seq_data={group_id: seq_data},
103 + sampling_params=sampling_params,
104 + block_tables=None,
105 + )
106 + seqs.append(seq)
107 +
108 + input_tokens, input_positions, input_metadata = self.prepare_inputs(seqs)
109 +
110 + # Execute the model.
111 + self.model(
112 + input_ids=input_tokens,
113 + positions=input_positions,
114 + kv_caches=[(None, None)] * self.num_layers,
115 + input_metadata=input_metadata,
116 + cache_events=None,
117 + )
118 +
119 + # Calculate the number of blocks that can be allocated with the
120 + # profiled peak memory.
121 + torch.cuda.synchronize()
122 + peak_memory = torch.cuda.max_memory_allocated()
123 + total_gpu_memory = get_gpu_memory()
124 + cache_block_size = get_cache_block_size(block_size, self.num_heads,
125 + self.head_size, self.num_layers,
126 + self.dtype)
127 + num_gpu_blocks = int((total_gpu_memory * gpu_memory_utilization
128 + - peak_memory) // cache_block_size)
129 + num_cpu_blocks = int(cpu_swap_space // cache_block_size)
130 + torch.cuda.empty_cache()
131 + # Reset the seed to ensure that the model output is not affected by
132 + # the profiling.
133 + set_random_seed(self.seed)
134 + return num_gpu_blocks, num_cpu_blocks
135 +
136 + def init_cache_engine(self, block_size: int, num_gpu_blocks: int,
137 + num_cpu_blocks: int):
138 + self.block_size = block_size
139 self.cache_engine = CacheEngine(
140 worker_id=self.worker_id,
141 num_layers=self.num_layers,
142 num_heads=self.num_heads,
143 head_size=self.head_size,
74 - block_size=block_size,
144 + block_size=self.block_size,
145 num_gpu_blocks=num_gpu_blocks,
146 num_cpu_blocks=num_cpu_blocks,
147 dtype=self.dtype,
@@ -129,6 +199,12 @@ class Worker:
199 # is always the first token in the sequence.
200 input_positions.extend(range(len(prompt_tokens)))
201
202 + if seq_group_metadata.block_tables is None:
203 + # During memory profiling, the block tables are not initialized
204 + # yet. In this case, we just use a dummy slot mapping.
205 + slot_mapping.extend([0] * prompt_len)
206 + continue
207 +
208 # Compute the slot mapping.
209 block_table = seq_group_metadata.block_tables[seq_id]
210 for i in range(prompt_len):