A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Use runtime profiling to replace manual memory analyzers (#81)
Zhuohan Li committed
May 19, 2023 at 11:35 UTC
f756799b84f5558c82c7a049069f845b31573e9e
14 files changed
+211
-478
cacheflow/core/server.py
+30
-22
@@ -6,15 +6,14 @@ try:
6
import ray
7
except ImportError:
8
ray = None
9
+import numpy as np
10
import torch
11
12
from cacheflow.core.scheduler import Scheduler
13
from cacheflow.frontend.simple_frontend import SimpleFrontend
14
from cacheflow.logger import init_logger
14
-from cacheflow.model_executor import get_memory_analyzer
15
from cacheflow.sampling_params import SamplingParams
16
from cacheflow.sequence import SequenceGroup
17
-from cacheflow.utils import get_gpu_memory, get_cpu_memory
17
from cacheflow.worker.controller import Controller, DeviceID
18
19
logger = init_logger(__name__)
@@ -34,14 +33,13 @@ class Server:
33
dtype: str,
34
seed: int,
35
swap_space: int,
36
+ gpu_memory_utilization: float,
37
max_num_batched_tokens: int,
38
max_num_sequences: int,
39
num_nodes: int,
40
num_devices_per_node: int,
41
distributed_init_method: str,
42
all_stage_devices: List[List[DeviceID]],
43
- gpu_memory: int,
44
- cpu_memory: int,
43
use_ray: bool,
44
log_stats: bool,
45
):
@@ -63,19 +61,6 @@ class Server:
61
assert self.world_size == 1, (
62
"Only support single GPU without Ray.")
63
66
- self.memory_analyzer = get_memory_analyzer(
67
- model_name=model,
68
- block_size=block_size,
69
- dtype=dtype,
70
- gpu_memory=gpu_memory,
71
- cpu_memory=cpu_memory,
72
- tensor_parallel_size=tensor_parallel_size,
73
- )
74
- self.num_gpu_blocks = self.memory_analyzer.get_max_num_gpu_blocks(
75
- max_num_batched_tokens=max_num_batched_tokens)
76
- self.num_cpu_blocks = self.memory_analyzer.get_max_num_cpu_blocks(
77
- swap_space_gib=swap_space)
78
-
64
# Create a controller for each pipeline stage.
65
self.controllers: List[Controller] = []
66
for i in range(pipeline_parallel_size):
@@ -87,19 +72,35 @@ class Server:
72
tensor_parallel_size=tensor_parallel_size,
73
distributed_init_method=distributed_init_method,
74
model_name=model,
90
- block_size=block_size,
91
- num_gpu_blocks=self.num_gpu_blocks,
92
- num_cpu_blocks=self.num_cpu_blocks,
75
dtype=dtype,
76
seed=seed,
77
cache_dir=cache_dir,
78
use_dummy_weights=use_dummy_weights,
79
use_np_cache=use_np_cache,
80
max_num_batched_tokens=max_num_batched_tokens,
81
+ max_num_sequences=max_num_sequences,
82
use_ray=use_ray,
83
)
84
self.controllers.append(controller)
85
86
+ # Initialize cache engine.
87
+ all_worker_num_available_blocks = []
88
+ for controller in self.controllers:
89
+ all_worker_num_available_blocks.extend(
90
+ controller.get_num_available_blocks(
91
+ block_size, swap_space, gpu_memory_utilization)
92
+ )
93
+ # Since we use a shared centralized controller, we take the minimum
94
+ # number of blocks across all workers to make sure all the memory
95
+ # operators can be applied to all workers.
96
+ self.num_gpu_blocks = np.min([b[0] for b in all_worker_num_available_blocks])
97
+ self.num_cpu_blocks = np.min([b[1] for b in all_worker_num_available_blocks])
98
+ logger.info(f'# GPU blocks: {self.num_gpu_blocks}, '
99
+ f'# CPU blocks: {self.num_cpu_blocks}')
100
+ for controller in self.controllers:
101
+ controller.init_cache_engine(block_size, self.num_gpu_blocks,
102
+ self.num_cpu_blocks)
103
+
104
# Create a scheduler.
105
self.scheduler = Scheduler(
106
controllers=self.controllers,
@@ -214,7 +215,11 @@ def initialize_cluster(
215
all_stage_devices)
216
217
218
+_GiB = 1 << 30
219
+
220
+
221
def add_server_arguments(parser: argparse.ArgumentParser):
222
+ """Shared arguments for CacheFlow servers."""
223
# Model arguments
224
parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
225
parser.add_argument('--cache-dir', type=str, default=None,
@@ -238,6 +243,7 @@ def add_server_arguments(parser: argparse.ArgumentParser):
243
# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
244
parser.add_argument('--seed', type=int, default=0, help='random seed')
245
parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
246
+ parser.add_argument('--gpu-memory-utilization', type=float, default=0.95, help='the percentage of GPU memory to be used for the model executor')
247
parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens per iteration')
248
parser.add_argument('--max-num-sequences', type=int, default=256, help='maximum number of sequences per iteration')
249
parser.add_argument('--log-stats', action='store_true', help='log system statistics')
@@ -245,8 +251,11 @@ def add_server_arguments(parser: argparse.ArgumentParser):
251
252
253
def process_server_arguments(args: argparse.Namespace):
254
+ """Post process the parsed arguments."""
255
if args.pipeline_parallel_size * args.tensor_parallel_size > 1:
256
args.use_ray = True
257
+ args.swap_space = args.swap_space * _GiB
258
+ args.max_num_sequences = min(args.max_num_sequences, args.max_num_batched_tokens)
259
return args
260
261
@@ -274,14 +283,13 @@ def init_local_server_and_frontend_with_arguments(args: argparse.Namespace):
283
dtype=args.dtype,
284
seed=args.seed,
285
swap_space=args.swap_space,
286
+ gpu_memory_utilization=args.gpu_memory_utilization,
287
max_num_batched_tokens=args.max_num_batched_tokens,
288
max_num_sequences=args.max_num_sequences,
289
num_nodes=num_nodes,
290
num_devices_per_node=num_devices_per_node,
291
distributed_init_method=distributed_init_method,
292
all_stage_devices=all_stage_devices,
283
- gpu_memory=get_gpu_memory(),
284
- cpu_memory=get_cpu_memory(),
293
use_ray=args.use_ray,
294
log_stats=args.log_stats,
295
)
cacheflow/frontend/fastapi_frontend.py
+7
-3
@@ -15,7 +15,7 @@ from cacheflow.core.server import (Server, add_server_arguments,
15
from cacheflow.frontend.utils import get_tokenizer
16
from cacheflow.sampling_params import SamplingParams
17
from cacheflow.sequence import Sequence, SequenceGroup
18
-from cacheflow.utils import Counter, get_gpu_memory, get_cpu_memory
18
+from cacheflow.utils import Counter
19
from cacheflow.worker.controller import DeviceID
20
21
TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
@@ -34,6 +34,7 @@ class FastAPIServer:
34
dtype: str,
35
seed: int,
36
swap_space: int,
37
+ gpu_memory_utilization: float,
38
max_num_batched_tokens: int,
39
max_num_sequences: int,
40
num_nodes: int,
@@ -41,6 +42,7 @@ class FastAPIServer:
42
distributed_init_method: str,
43
all_stage_devices: List[List[DeviceID]],
44
server_use_ray: bool,
45
+ log_stats: bool,
46
):
47
self.block_size = block_size
48
@@ -62,15 +64,15 @@ class FastAPIServer:
64
dtype=dtype,
65
seed=seed,
66
swap_space=swap_space,
67
+ gpu_memory_utilization=gpu_memory_utilization,
68
max_num_batched_tokens=max_num_batched_tokens,
69
max_num_sequences=max_num_sequences,
70
num_nodes=num_nodes,
71
num_devices_per_node=num_devices_per_node,
72
distributed_init_method=distributed_init_method,
73
all_stage_devices=all_stage_devices,
71
- gpu_memory=get_gpu_memory(),
72
- cpu_memory=get_cpu_memory(),
74
use_ray=server_use_ray,
75
+ log_stats=log_stats,
76
)
77
78
self.running_seq_groups: Dict[int, SequenceGroup] = {}
@@ -182,6 +184,7 @@ if __name__ == "__main__":
184
dtype=args.dtype,
185
seed=args.seed,
186
swap_space=args.swap_space,
187
+ gpu_memory_utilization=args.gpu_memory_utilization,
188
max_num_batched_tokens=args.max_num_batched_tokens,
189
max_num_sequences=args.max_num_sequences,
190
num_nodes=num_nodes,
@@ -189,6 +192,7 @@ if __name__ == "__main__":
192
distributed_init_method=distributed_init_method,
193
all_stage_devices=all_stage_devices,
194
server_use_ray=args.use_ray,
195
+ log_stats=args.log_stats,
196
)
197
198
uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/model_executor/__init__.py
+4
-3
@@ -1,11 +1,12 @@
1
from cacheflow.model_executor.input_metadata import InputMetadata
2
-from cacheflow.model_executor.model_loader import get_model, get_memory_analyzer
3
-from cacheflow.model_executor.utils import set_random_seed
2
+from cacheflow.model_executor.model_loader import get_model
3
+from cacheflow.model_executor.utils import (set_random_seed,
4
+ get_cache_block_size)
5
6
7
__all__ = [
8
"InputMetadata",
9
+ "get_cache_block_size",
10
"get_model",
9
- "get_memory_analyzer",
11
"set_random_seed",
12
]
cacheflow/model_executor/layers/attention.py
+29
-21
@@ -11,6 +11,8 @@ from cacheflow import pos_encoding_ops
11
from cacheflow.model_executor.input_metadata import InputMetadata
12
13
14
+_SUPPORTED_HEAD_SIZES = [32, 64, 80, 96, 128, 160, 192, 256]
15
+
16
class GPTCacheFlowAttention(nn.Module):
17
"""GPT-style multi-head attention.
18
@@ -39,11 +41,19 @@ class GPTCacheFlowAttention(nn.Module):
41
5. Output a flattened 1D tensor.
42
"""
43
42
- def __init__(self, scale: float) -> None:
44
+ def __init__(self, num_heads: int, head_size: int, scale: float) -> None:
45
super().__init__()
46
+ self.num_heads = num_heads
47
+ self.head_size = head_size
48
self.scale = float(scale)
49
self.attn_op = xops.fmha.cutlass.FwOp()
50
51
+ if self.head_size not in _SUPPORTED_HEAD_SIZES:
52
+ raise ValueError(f'head_size ({self.head_size}) is not supported by '
53
+ 'the single_query_cached_kv_attention kernel. '
54
+ 'Use one of the following head sizes: '
55
+ f'{_SUPPORTED_HEAD_SIZES}.')
56
+
57
def multi_query_kv_attention(
58
self,
59
output: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
@@ -74,14 +84,6 @@ class GPTCacheFlowAttention(nn.Module):
84
value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
85
input_metadata: InputMetadata,
86
) -> None:
77
- head_size = value_cache.shape[2]
78
- supported_head_sizes = [32, 64, 80, 96, 128, 160, 192, 256]
79
- if head_size not in supported_head_sizes:
80
- raise ValueError(f'head_size ({head_size}) is not supported by '
81
- 'the single_query_cached_kv_attention kernel. '
82
- 'Use one of the following head sizes: '
83
- f'{supported_head_sizes}.')
84
-
87
block_size = value_cache.shape[3]
88
attention_ops.single_query_cached_kv_attention(
89
output,
@@ -100,8 +102,8 @@ class GPTCacheFlowAttention(nn.Module):
102
query: torch.Tensor, # [num_tokens, num_heads * head_size]
103
key: torch.Tensor, # [num_tokens, num_heads * head_size]
104
value: torch.Tensor, # [num_tokens, num_heads * head_size]
103
- key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
104
- value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
105
+ key_cache: Optional[torch.Tensor], # [num_blocks, num_heads, head_size/x, block_size, x]
106
+ value_cache: Optional[torch.Tensor], # [num_blocks, num_heads, head_size, block_size]
107
input_metadata: InputMetadata,
108
cache_event: Optional[torch.cuda.Event],
109
) -> torch.Tensor: # [num_tokens, num_heads * head_size]
@@ -109,11 +111,9 @@ class GPTCacheFlowAttention(nn.Module):
111
# tensor of shape [num_tokens, 3 * num_heads * head_size].
112
113
# Reshape the query, key, and value tensors.
112
- num_heads = value_cache.shape[1]
113
- head_size = value_cache.shape[2]
114
- query = query.view(-1, num_heads, head_size)
115
- key = key.view(-1, num_heads, head_size)
116
- value = value.view(-1, num_heads, head_size)
114
+ query = query.view(-1, self.num_heads, self.head_size)
115
+ key = key.view(-1, self.num_heads, self.head_size)
116
+ value = value.view(-1, self.num_heads, self.head_size)
117
118
# Pre-allocate the output tensor.
119
output = torch.empty_like(query)
@@ -134,8 +134,11 @@ class GPTCacheFlowAttention(nn.Module):
134
cache_event.wait()
135
136
# Reshape the keys and values and store them in the cache.
137
+ # When key_cache and value_cache are not provided, the new key
138
+ # and value vectors will not be cached.
139
num_valid_tokens = input_metadata.num_valid_tokens
138
- if num_valid_tokens > 0:
140
+ if (num_valid_tokens > 0 and key_cache is not None
141
+ and value_cache is not None):
142
# The stride is 3 because the key and value are sliced from qkv.
143
cache_ops.reshape_and_cache(
144
key[:num_valid_tokens],
@@ -146,6 +149,10 @@ class GPTCacheFlowAttention(nn.Module):
149
)
150
151
if input_metadata.num_generation_tokens > 0:
152
+ assert key_cache is not None and value_cache is not None, (
153
+ "key_cache and value_cache must be provided when "
154
+ "generating tokens."
155
+ )
156
# Compute the attention op for generation tokens.
157
self.single_query_cached_kv_attention(
158
output[num_prompt_tokens:num_valid_tokens],
@@ -156,7 +163,7 @@ class GPTCacheFlowAttention(nn.Module):
163
164
# Reshape the output tensor.
165
# NOTE(woosuk): The output tensor may include paddings.
159
- return output.view(-1, num_heads * head_size)
166
+ return output.view(-1, self.num_heads * self.head_size)
167
168
169
class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
@@ -164,12 +171,14 @@ class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
171
172
def __init__(
173
self,
174
+ num_heads: int,
175
+ head_size: int,
176
scale: float,
177
rotary_dim: int,
178
max_position: int = 8192,
179
base: int = 10000,
180
) -> None:
172
- super().__init__(scale)
181
+ super().__init__(num_heads, head_size, scale)
182
183
# Create the cos and sin cache.
184
inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2) / rotary_dim))
@@ -199,12 +208,11 @@ class GPTNeoXCacheFlowAttention(GPTCacheFlowAttention):
208
) -> torch.Tensor: # [num_tokens, num_heads * head_size]
209
# Apply rotary embedding to the query and key before passing them
210
# to the attention op.
202
- head_size = value_cache.shape[2]
211
pos_encoding_ops.rotary_embedding_neox(
212
positions,
213
query,
214
key,
207
- head_size,
215
+ self.head_size,
216
self.cos_sin_cache,
217
)
218
return super().forward(
cacheflow/model_executor/layers/sampler.py
+1
-1
@@ -74,7 +74,7 @@ class Sampler(nn.Module):
74
# Apply top-p and top-k truncation.
75
top_ps, top_ks = _get_top_p_top_k(input_metadata, self.vocab_size)
76
assert len(top_ps) == len(top_ks) == probs.shape[0]
77
- if any(p < 1.0 for p in top_ps) or any(k != -1 for k in top_ks):
77
+ if any(p < 1.0 for p in top_ps) or any(k != self.vocab_size for k in top_ks):
78
probs = _apply_top_p_top_k(probs, top_ps, top_ks)
79
80
# Sample the next tokens.
cacheflow/model_executor/memory_analyzer.py
deleted
-370
@@ -1,370 +0,0 @@
1
-import torch
2
-from transformers import AutoConfig
3
-
4
-from cacheflow.logger import init_logger
5
-from cacheflow.model_executor.utils import get_dtype_size
6
-
7
-logger = init_logger(__name__)
8
-
9
-_GiB = 1 << 30
10
-
11
-
12
-class CacheFlowMemoryAnalyzer:
13
-
14
- def get_max_num_gpu_blocks(
15
- self,
16
- max_num_batched_tokens: int,
17
- memory_utilization: float,
18
- ) -> int:
19
- raise NotImplementedError()
20
-
21
- def get_workspace_size(self) -> int:
22
- return 1 * _GiB
23
-
24
- def get_cache_block_size(self) -> int:
25
- raise NotImplementedError()
26
-
27
- def get_max_num_cpu_blocks(
28
- self,
29
- swap_space_gib: int,
30
- ) -> int:
31
- swap_space = swap_space_gib * _GiB
32
- cpu_memory = self.cpu_memory
33
- if swap_space > 0.8 * cpu_memory:
34
- raise ValueError(f'The swap space ({swap_space_gib:.2f} GiB) '
35
- 'takes more than 80% of the available memory '
36
- f'({cpu_memory / _GiB:.2f} GiB).'
37
- 'Please check the swap space size.')
38
- if swap_space > 0.5 * cpu_memory:
39
- logger.info(f'WARNING: The swap space ({swap_space_gib:.2f} GiB) '
40
- 'takes more than 50% of the available memory '
41
- f'({cpu_memory / _GiB:.2f} GiB).'
42
- 'This may slow the system performance.')
43
- max_num_blocks = swap_space // self.get_cache_block_size()
44
- return max_num_blocks
45
-
46
- def get_param_size(self) -> int:
47
- raise NotImplementedError()
48
-
49
- def get_max_act_size(self, max_num_batched_tokens: int) -> int:
50
- raise NotImplementedError()
51
-
52
- def get_cache_block_size(self) -> int:
53
- key_cache_block = self.block_size * self.hidden_size // self.tensor_parallel_size
54
- value_cache_block = key_cache_block
55
- total = self.num_layers * (key_cache_block + value_cache_block)
56
- dtype_size = get_dtype_size(self.dtype)
57
- return dtype_size * total
58
-
59
- def get_max_num_gpu_blocks(
60
- self,
61
- max_num_batched_tokens: int,
62
- memory_utilization: float = 0.95,
63
- ) -> int:
64
- # NOTE(woosuk): This assumes that the machine has homogeneous GPUs.
65
- usable_memory = int(memory_utilization * self.gpu_memory)
66
-
67
- param_size = self.get_param_size()
68
- act_size = self.get_max_act_size(max_num_batched_tokens)
69
- workspace_size = self.get_workspace_size()
70
-
71
- max_cache_size = usable_memory - (param_size + act_size + workspace_size)
72
- if max_cache_size <= 0:
73
- raise RuntimeError('Not enough GPU memory.')
74
- max_num_blocks = max_cache_size // self.get_cache_block_size()
75
- return max_num_blocks
76
-
77
-
78
-class GPT2MemoryAnalyzer(CacheFlowMemoryAnalyzer):
79
-
80
- def __init__(
81
- self,
82
- model_name: str,
83
- block_size: int,
84
- dtype: torch.dtype,
85
- gpu_memory: int,
86
- cpu_memory: int,
87
- tensor_parallel_size: int,
88
- ) -> None:
89
- self.model_name = model_name
90
- self.block_size = block_size
91
- self.dtype = dtype
92
- self.gpu_memory = gpu_memory
93
- self.cpu_memory = cpu_memory
94
- self.tensor_parallel_size = tensor_parallel_size
95
-
96
- config = AutoConfig.from_pretrained(model_name)
97
- self.num_layers = config.num_hidden_layers
98
- self.hidden_size = config.hidden_size
99
- self.num_heads = config.num_attention_heads
100
- self.head_size = config.hidden_size // self.num_heads
101
- self.ffn_size = config.n_inner if config.n_inner is not None else 4 * self.hidden_size
102
- self.vocab_size = config.vocab_size
103
- self.max_position = config.max_position_embeddings
104
-
105
- def get_param_size(self) -> int:
106
- word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
107
- position_embedding = self.max_position * self.hidden_size
108
-
109
- ln1 = 2 * self.hidden_size
110
- q = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
111
- k = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
112
- v = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
113
- out = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
114
- mha = ln1 + q + k + v + out
115
-
116
- ln2 = 2 * self.hidden_size
117
- ffn1 = self.hidden_size * self.ffn_size // self.tensor_parallel_size + self.ffn_size
118
- ffn2 = self.ffn_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
119
- ffn = ln2 + ffn1 + ffn2
120
-
121
- total = (word_embedding + position_embedding +
122
- self.num_layers * (mha + ffn))
123
- dtype_size = get_dtype_size(self.dtype)
124
- return dtype_size * total
125
-
126
- def get_max_act_size(
127
- self,
128
- max_num_batched_tokens: int,
129
- ) -> int:
130
- # NOTE: We approxmiately calculate the maximum activation size by
131
- # estimating
132
- # 1) the maximum activation tensor size during inference
133
- # 2) the residual tensor size during inference
134
- # Here, we assume that FlashAttention is used and
135
- # thus the attention maps are never materialized in GPU DRAM.
136
- residual = max_num_batched_tokens * self.hidden_size
137
- qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
138
- ffn = max_num_batched_tokens * self.ffn_size // self.tensor_parallel_size
139
- # Double the activation size for input and output.
140
- max_act = 2 * (max(qkv, ffn) + residual)
141
- # Size of output logits.
142
- output_logits = 2 * (max_num_batched_tokens * self.vocab_size)
143
- max_act = max(max_act, output_logits)
144
- dtype_size = get_dtype_size(self.dtype)
145
- return dtype_size * max_act
146
-
147
-
148
-class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
149
-
150
- def __init__(
151
- self,
152
- model_name: str,
153
- block_size: int,
154
- dtype: torch.dtype,
155
- gpu_memory: int,
156
- cpu_memory: int,
157
- tensor_parallel_size: int,
158
- ) -> None:
159
- self.model_name = model_name
160
- self.block_size = block_size
161
- self.dtype = dtype
162
- self.gpu_memory = gpu_memory
163
- self.cpu_memory = cpu_memory
164
- self.tensor_parallel_size = tensor_parallel_size
165
-
166
- config = AutoConfig.from_pretrained(model_name)
167
- self.num_layers = config.num_hidden_layers
168
- self.hidden_size = config.hidden_size
169
- self.num_heads = config.num_attention_heads
170
- self.head_size = config.hidden_size // self.num_heads
171
- self.ffn_size = config.ffn_dim
172
- self.embedding_size = config.word_embed_proj_dim
173
- self.vocab_size = config.vocab_size
174
- self.max_position = config.max_position_embeddings
175
-
176
- def get_param_size(self) -> int:
177
- word_embedding = self.vocab_size * self.embedding_size // self.tensor_parallel_size
178
- if self.embedding_size != self.hidden_size:
179
- # Project in/out.
180
- word_embedding += 2 * self.embedding_size * self.hidden_size
181
- position_embedding = self.max_position * self.hidden_size
182
-
183
- ln1 = 2 * self.hidden_size
184
- q = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
185
- k = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
186
- v = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
187
- out = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
188
- mha = ln1 + q + k + v + out
189
-
190
- ln2 = 2 * self.hidden_size
191
- ffn1 = self.hidden_size * self.ffn_size // self.tensor_parallel_size + self.ffn_size
192
- ffn2 = self.ffn_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
193
- ffn = ln2 + ffn1 + ffn2
194
-
195
- total = (word_embedding + position_embedding +
196
- self.num_layers * (mha + ffn))
197
- dtype_size = get_dtype_size(self.dtype)
198
- return dtype_size * total
199
-
200
- def get_max_act_size(
201
- self,
202
- max_num_batched_tokens: int,
203
- ) -> int:
204
- # NOTE: We approxmiately calculate the maximum activation size by
205
- # estimating
206
- # 1) the maximum activation tensor size during inference
207
- # 2) the residual tensor size during inference
208
- # Here, we assume that we use memory-efficient attention which
209
- # does not materialize the attention maps in GPU DRAM.
210
- residual = max_num_batched_tokens * self.hidden_size
211
- qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
212
- ffn = max_num_batched_tokens * self.ffn_size // self.tensor_parallel_size
213
- # Double the activation size for input and output.
214
- max_act = 2 * (max(qkv, ffn) + residual)
215
- # Size of output logits.
216
- output_logits = 2 * (max_num_batched_tokens * self.vocab_size)
217
- max_act = max(max_act, output_logits)
218
- dtype_size = get_dtype_size(self.dtype)
219
- return dtype_size * max_act
220
-
221
-
222
-class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
223
-
224
- def __init__(
225
- self,
226
- model_name: str,
227
- block_size: int,
228
- dtype: torch.dtype,
229
- gpu_memory: int,
230
- cpu_memory: int,
231
- tensor_parallel_size: int,
232
- ) -> None:
233
- self.model_name = model_name
234
- self.block_size = block_size
235
- self.dtype = dtype
236
- self.gpu_memory = gpu_memory
237
- self.cpu_memory = cpu_memory
238
- self.tensor_parallel_size = tensor_parallel_size
239
-
240
- config = AutoConfig.from_pretrained(model_name)
241
- self.num_layers = config.num_hidden_layers
242
- self.hidden_size = config.hidden_size
243
- self.num_heads = config.num_attention_heads
244
- self.head_size = config.hidden_size // self.num_heads
245
- self.ffn_size = config.intermediate_size
246
- self.vocab_size = config.vocab_size
247
- self.max_position = 8192
248
-
249
- def get_param_size(self) -> int:
250
- # NOTE: LLaMA does not tie the two embeddings.
251
- word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
252
- lm_head = self.vocab_size * self.hidden_size // self.tensor_parallel_size
253
-
254
- # NOTE: LLaMA does not have bias terms.
255
- ln1 = self.hidden_size
256
- q = self.hidden_size * self.hidden_size // self.tensor_parallel_size
257
- k = self.hidden_size * self.hidden_size // self.tensor_parallel_size
258
- v = self.hidden_size * self.hidden_size // self.tensor_parallel_size
259
- out = self.hidden_size * self.hidden_size // self.tensor_parallel_size
260
- # Rotary embedding.
261
- # TODO(woosuk): Share the rotary embedding between layers.
262
- rot = self.max_position * self.head_size
263
- mha = ln1 + q + k + v + out + rot
264
-
265
- ln2 = self.hidden_size
266
- gate = self.hidden_size * self.ffn_size // self.tensor_parallel_size
267
- down = self.ffn_size * self.hidden_size // self.tensor_parallel_size
268
- up = self.hidden_size * self.ffn_size // self.tensor_parallel_size
269
- ffn = ln2 + gate + down + up
270
-
271
- total = word_embedding + self.num_layers * (mha + ffn) + lm_head
272
- dtype_size = get_dtype_size(self.dtype)
273
- return dtype_size * total
274
-
275
- def get_max_act_size(
276
- self,
277
- max_num_batched_tokens: int,
278
- ) -> int:
279
- # NOTE: We approxmiately calculate the maximum activation size by
280
- # estimating
281
- # 1) the maximum activation tensor size during inference
282
- # 2) the residual tensor size during inference
283
- # Here, we assume that we use memory-efficient attention which
284
- # does not materialize the attention maps in GPU DRAM.
285
- residual = max_num_batched_tokens * self.hidden_size
286
- qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
287
- ffn = 2 * (max_num_batched_tokens * self.ffn_size) // self.tensor_parallel_size
288
- # Double the activation size for input and output.
289
- max_act = 2 * (max(qkv, ffn) + residual)
290
- # Size of output logits.
291
- output_logits = 2 * (max_num_batched_tokens * self.vocab_size)
292
- max_act = max(max_act, output_logits)
293
- dtype_size = get_dtype_size(self.dtype)
294
- return dtype_size * max_act
295
-
296
-
297
-class GPTNeoXMemoryAnalyzer(CacheFlowMemoryAnalyzer):
298
-
299
- def __init__(
300
- self,
301
- model_name: str,
302
- block_size: int,
303
- dtype: torch.dtype,
304
- gpu_memory: int,
305
- cpu_memory: int,
306
- tensor_parallel_size: int,
307
- ) -> None:
308
- self.model_name = model_name
309
- self.block_size = block_size
310
- self.dtype = dtype
311
- self.gpu_memory = gpu_memory
312
- self.cpu_memory = cpu_memory
313
- self.tensor_parallel_size = tensor_parallel_size
314
-
315
- config = AutoConfig.from_pretrained(model_name)
316
- self.num_layers = config.num_hidden_layers
317
- self.hidden_size = config.hidden_size
318
- self.num_heads = config.num_attention_heads
319
- self.head_size = config.hidden_size // self.num_heads
320
- self.ffn_size = config.intermediate_size
321
- self.vocab_size = config.vocab_size
322
- self.max_position = 8192
323
- self.tie_word_embeddings = config.tie_word_embeddings
324
-
325
- def get_param_size(self) -> int:
326
- word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
327
- if self.tie_word_embeddings:
328
- lm_head = 0
329
- else:
330
- lm_head = self.vocab_size * self.hidden_size // self.tensor_parallel_size
331
-
332
- ln1 = 2 * self.hidden_size
333
- q = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
334
- k = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
335
- v = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
336
- out = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
337
- # Rotary embedding.
338
- # TODO(woosuk): Share the rotary embedding between layers.
339
- rot = self.max_position * self.head_size
340
- mha = ln1 + q + k + v + out + rot
341
-
342
- ln2 = 2 * self.hidden_size
343
- ffn1 = self.hidden_size * self.ffn_size // self.tensor_parallel_size + self.ffn_size
344
- ffn2 = self.ffn_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
345
- ffn = ln2 + ffn1 + ffn2
346
-
347
- total = word_embedding + self.num_layers * (mha + ffn) + lm_head
348
- dtype_size = get_dtype_size(self.dtype)
349
- return dtype_size * total
350
-
351
- def get_max_act_size(
352
- self,
353
- max_num_batched_tokens: int,
354
- ) -> int:
355
- # NOTE: We approxmiately calculate the maximum activation size by
356
- # estimating
357
- # 1) the maximum activation tensor size during inference
358
- # 2) the residual tensor size during inference
359
- # Here, we assume that we use memory-efficient attention which
360
- # does not materialize the attention maps in GPU DRAM.
361
- residual = max_num_batched_tokens * self.hidden_size
362
- qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
363
- ffn = 2 * (max_num_batched_tokens * self.ffn_size) // self.tensor_parallel_size
364
- # Double the activation size for input and output.
365
- max_act = 2 * (max(qkv, ffn) + residual)
366
- # Size of output logits.
367
- output_logits = 2 * (max_num_batched_tokens * self.vocab_size)
368
- max_act = max(max_act, output_logits)
369
- dtype_size = get_dtype_size(self.dtype)
370
- return dtype_size * max_act
cacheflow/model_executor/model_loader.py
-37
@@ -5,9 +5,6 @@ import torch
5
import torch.nn as nn
6
from transformers import AutoConfig, PretrainedConfig
7
8
-from cacheflow.model_executor.memory_analyzer import (
9
- CacheFlowMemoryAnalyzer, GPT2MemoryAnalyzer, GPTNeoXMemoryAnalyzer,
10
- LlamaMemoryAnalyzer, OPTMemoryAnalyzer)
8
from cacheflow.model_executor.models import (
9
GPT2LMHeadModel, GPTNeoXForCausalLM, LlamaForCausalLM, OPTForCausalLM)
10
from cacheflow.model_executor.utils import get_torch_dtype
@@ -22,14 +19,6 @@ _MODEL_REGISTRY = {
19
"OPTForCausalLM": OPTForCausalLM,
20
}
21
25
-_MEMORY_ANALYZERS = {
26
- "GPT2LMHeadModel": GPT2MemoryAnalyzer,
27
- "GPTNeoXForCausalLM": GPTNeoXMemoryAnalyzer,
28
- "LlamaForCausalLM": LlamaMemoryAnalyzer,
29
- "OPTForCausalLM": OPTMemoryAnalyzer,
30
-}
31
-
32
-
22
def _get_model_architecture(config: PretrainedConfig) -> nn.Module:
23
architectures = getattr(config, "architectures", [])
24
for arch in architectures:
@@ -41,17 +30,6 @@ def _get_model_architecture(config: PretrainedConfig) -> nn.Module:
30
)
31
32
44
-def _get_memory_analyzer(config: PretrainedConfig) -> CacheFlowMemoryAnalyzer:
45
- architectures = getattr(config, "architectures", [])
46
- for arch in architectures:
47
- if arch in _MEMORY_ANALYZERS:
48
- return _MEMORY_ANALYZERS[arch]
49
- raise ValueError(
50
- f"Model architectures {architectures} are not supported for now. "
51
- f"Supported architectures: {list(_MEMORY_ANALYZERS.keys())}"
52
- )
53
-
54
-
33
def _get_dtype(config: PretrainedConfig, dtype: str) -> torch.dtype:
34
# NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
35
# because config.torch_dtype can be None.
@@ -100,18 +78,3 @@ def get_model(
78
model = model.cuda()
79
return model.eval(), torch_dtype
80
103
-
104
-def get_memory_analyzer(
105
- model_name: str,
106
- block_size: int,
107
- dtype: str,
108
- gpu_memory: int,
109
- cpu_memory: int,
110
- tensor_parallel_size: int = 1,
111
-) -> CacheFlowMemoryAnalyzer:
112
- config = AutoConfig.from_pretrained(model_name)
113
- torch_dtype = _get_dtype(config, dtype)
114
- memory_analyzer = _get_memory_analyzer(config)
115
- return memory_analyzer(
116
- model_name, block_size, torch_dtype, gpu_memory, cpu_memory,
117
- tensor_parallel_size)
cacheflow/model_executor/models/gpt2.py
+2
-1
@@ -58,7 +58,8 @@ class GPT2Attention(nn.Module):
58
self.c_proj = RowParallelLinear(self.hidden_size, self.hidden_size, bias=True,
59
input_is_parallel=True,
60
perform_initialization=False)
61
- self.attn = GPTCacheFlowAttention(scale=self.scale)
61
+ self.attn = GPTCacheFlowAttention(self.num_heads, self.head_dim,
62
+ scale=self.scale)
63
64
def forward(
65
self,
cacheflow/model_executor/models/gpt_neox.py
+2
-1
@@ -62,7 +62,8 @@ class GPTNeoXAttention(nn.Module):
62
scaling = self.head_size ** -0.5
63
rotary_dim = int(self.head_size * config.rotary_pct)
64
assert rotary_dim % 2 == 0
65
- self.attn = GPTNeoXCacheFlowAttention(scaling, rotary_dim)
65
+ self.attn = GPTNeoXCacheFlowAttention(self.num_heads, self.head_size,
66
+ scaling, rotary_dim)
67
68
def forward(
69
self,
cacheflow/model_executor/models/llama.py
+2
-1
@@ -104,7 +104,8 @@ class LlamaAttention(nn.Module):
104
input_is_parallel=True,
105
perform_initialization=False,
106
)
107
- self.attn = GPTNeoXCacheFlowAttention(self.scaling, self.head_dim)
107
+ self.attn = GPTNeoXCacheFlowAttention(self.num_heads, self.head_dim,
108
+ self.scaling, rotary_dim=self.head_dim)
109
110
def forward(
111
self,
cacheflow/model_executor/models/opt.py
+2
-1
@@ -74,7 +74,8 @@ class OPTAttention(nn.Module):
74
self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
75
input_is_parallel=True,
76
perform_initialization=False)
77
- self.attn = GPTCacheFlowAttention(scale=self.scaling)
77
+ self.attn = GPTCacheFlowAttention(self.num_heads, self.head_dim,
78
+ scale=self.scaling)
79
80
def forward(
81
self,
cacheflow/model_executor/utils.py
+12
@@ -40,3 +40,15 @@ def set_random_seed(seed: int) -> None:
40
41
if model_parallel_is_initialized():
42
model_parallel_cuda_manual_seed(seed)
43
+
44
+
45
+def get_cache_block_size(block_size: int,
46
+ num_heads: int,
47
+ head_size: int,
48
+ num_layers: int,
49
+ dtype: str) -> int:
50
+ key_cache_block = block_size * num_heads * head_size
51
+ value_cache_block = key_cache_block
52
+ total = num_layers * (key_cache_block + value_cache_block)
53
+ dtype_size = get_dtype_size(dtype)
54
+ return dtype_size * total
cacheflow/worker/controller.py
+36
-9
@@ -23,23 +23,18 @@ class Controller:
23
pipeline_parallel_size: int,
24
distributed_init_method: str,
25
model_name: str,
26
- block_size: int,
27
- num_gpu_blocks: int,
28
- num_cpu_blocks: int,
26
dtype: str,
27
seed: int,
28
cache_dir: Optional[str],
29
use_dummy_weights: bool,
30
use_np_cache: bool,
31
max_num_batched_tokens: int,
32
+ max_num_sequences: int,
33
use_ray: bool,
34
) -> None:
35
self.stage_id = stage_id
36
self.stage_devices = stage_devices
37
self.model_name = model_name
40
- self.block_size = block_size
41
- self.num_gpu_blocks = num_gpu_blocks
42
- self.num_cpu_blocks = num_cpu_blocks
38
self.use_ray = use_ray
39
40
# Which pipeline stage is this node assigned to?
@@ -56,9 +51,6 @@ class Controller:
51
worker_cls = Worker
52
worker = worker_cls(
53
model_name=model_name,
59
- block_size=block_size,
60
- num_gpu_blocks=num_gpu_blocks,
61
- num_cpu_blocks=num_cpu_blocks,
54
dtype=dtype,
55
seed=seed,
56
distributed_init_method=distributed_init_method,
@@ -70,9 +62,44 @@ class Controller:
62
use_dummy_weights=use_dummy_weights,
63
use_np_cache=use_np_cache,
64
max_num_batched_tokens=max_num_batched_tokens,
65
+ max_num_sequences=max_num_sequences,
66
)
67
self.workers.append(worker)
68
69
+ def get_num_available_blocks(self, block_size: int, cpu_swap_space: int,
70
+ gpu_memory_utilization: float) -> List[Tuple[int, int]]:
71
+ all_worker_results = []
72
+ for worker in self.workers:
73
+ executor = worker.get_num_available_blocks
74
+ if self.use_ray:
75
+ executor = executor.remote
76
+
77
+ result = executor(
78
+ block_size,
79
+ cpu_swap_space,
80
+ gpu_memory_utilization,
81
+ )
82
+ all_worker_results.append(result)
83
+ if self.use_ray:
84
+ all_worker_results = ray.get(all_worker_results)
85
+ return all_worker_results
86
+
87
+ def init_cache_engine(self, block_size: int, num_gpu_blocks: int,
88
+ num_cpu_blocks: int):
89
+ all_worker_futures = []
90
+ for worker in self.workers:
91
+ executor = worker.init_cache_engine
92
+ if self.use_ray:
93
+ executor = executor.remote
94
+ future = executor(
95
+ block_size,
96
+ num_gpu_blocks,
97
+ num_cpu_blocks,
98
+ )
99
+ all_worker_futures.append(future)
100
+ if self.use_ray:
101
+ ray.get(all_worker_futures)
102
+
103
def set_next(
104
self,
105
next_node: Union['Controller', 'Scheduler'],
cacheflow/worker/worker.py
+84
-8
@@ -3,7 +3,8 @@ from typing import Dict, List, Optional, Tuple
3
4
import torch
5
6
-from cacheflow.model_executor import get_model, InputMetadata, set_random_seed
6
+from cacheflow.model_executor import (get_model, get_cache_block_size,
7
+ InputMetadata, set_random_seed)
8
from cacheflow.model_executor.parallel_utils.parallel_state import (
9
initialize_model_parallel,
10
initialize_all_reduce_launcher,
@@ -12,6 +13,7 @@ from cacheflow.sampling_params import SamplingParams
13
from cacheflow.sequence import (SequenceData, SequenceGroupMetadata,
14
SequenceOutputs)
15
from cacheflow.worker.cache_engine import CacheEngine
16
+from cacheflow.utils import get_gpu_memory
17
18
19
class Worker:
@@ -25,9 +27,6 @@ class Worker:
27
def __init__(
28
self,
29
model_name: str,
28
- block_size: int,
29
- num_gpu_blocks: int,
30
- num_cpu_blocks: int,
30
dtype: str,
31
seed: int,
32
distributed_init_method: str,
@@ -37,6 +36,7 @@ class Worker:
36
use_dummy_weights: bool,
37
use_np_cache: bool,
38
max_num_batched_tokens: int,
39
+ max_num_sequences: int,
40
tensor_parallel_size: int = 1,
41
pipeline_parallel_size: int = 1,
42
) -> None:
@@ -46,8 +46,8 @@ class Worker:
46
tensor_parallel_size,
47
pipeline_parallel_size)
48
self.worker_id = rank
49
- self.block_size = block_size
50
- set_random_seed(seed)
49
+ self.seed = seed
50
+ set_random_seed(self.seed)
51
52
# Initialize the model.
53
self.model, self.dtype = get_model(
@@ -55,8 +55,10 @@ class Worker:
55
use_dummy_weights=use_dummy_weights, use_np_cache=use_np_cache)
56
tensor_model_parallel_world_size = (
57
get_tensor_model_parallel_world_size())
58
+ self.max_num_batched_tokens = max_num_batched_tokens
59
initialize_all_reduce_launcher(
59
- max_num_batched_tokens, self.model.config.hidden_size, self.dtype)
60
+ self.max_num_batched_tokens, self.model.config.hidden_size, self.dtype)
61
+ self.max_num_sequences = max_num_sequences
62
self.num_layers = self.model.config.num_hidden_layers
63
assert self.model.config.num_attention_heads % tensor_model_parallel_world_size == 0
64
self.num_heads = self.model.config.num_attention_heads // tensor_model_parallel_world_size
@@ -66,12 +68,80 @@ class Worker:
68
# the random state is not affected by the model initialization.
69
set_random_seed(seed)
70
71
+ # Uninitialized cache engine. Will be initialized with
72
+ # self.init_cache_engine().
73
+ self.block_size = None
74
+ self.cache_engine = None
75
+ self.cache_events = None
76
+ self.gpu_cache = None
77
+
78
+ @torch.inference_mode()
79
+ def get_num_available_blocks(
80
+ self, block_size: int, cpu_swap_space: int,
81
+ gpu_memory_utilization: float) -> Tuple[int, int]:
82
+ # Profile the memory usage of the model and get the maximum number of
83
+ # cache blocks that can be allocated with the remaining free memory.
84
+ torch.cuda.empty_cache()
85
+ torch.cuda.reset_peak_memory_stats()
86
+
87
+ # Profile memory usage with max_num_sequences sequences and the total
88
+ # number of tokens equal to max_num_batched_tokens.
89
+
90
+ # Enable top-k sampling to reflect the accurate memory usage.
91
+ sampling_params = SamplingParams(top_p=0.99,
92
+ top_k=self.model.config.vocab_size - 1)
93
+ seqs = []
94
+ for group_id in range(self.max_num_sequences):
95
+ seq_len = (self.max_num_batched_tokens // self.max_num_sequences +
96
+ (group_id < self.max_num_batched_tokens %
97
+ self.max_num_sequences))
98
+ seq_data = SequenceData([0] * seq_len)
99
+ seq = SequenceGroupMetadata(
100
+ group_id=group_id,
101
+ is_prompt=True,
102
+ seq_data={group_id: seq_data},
103
+ sampling_params=sampling_params,
104
+ block_tables=None,
105
+ )
106
+ seqs.append(seq)
107
+
108
+ input_tokens, input_positions, input_metadata = self.prepare_inputs(seqs)
109
+
110
+ # Execute the model.
111
+ self.model(
112
+ input_ids=input_tokens,
113
+ positions=input_positions,
114
+ kv_caches=[(None, None)] * self.num_layers,
115
+ input_metadata=input_metadata,
116
+ cache_events=None,
117
+ )
118
+
119
+ # Calculate the number of blocks that can be allocated with the
120
+ # profiled peak memory.
121
+ torch.cuda.synchronize()
122
+ peak_memory = torch.cuda.max_memory_allocated()
123
+ total_gpu_memory = get_gpu_memory()
124
+ cache_block_size = get_cache_block_size(block_size, self.num_heads,
125
+ self.head_size, self.num_layers,
126
+ self.dtype)
127
+ num_gpu_blocks = int((total_gpu_memory * gpu_memory_utilization
128
+ - peak_memory) // cache_block_size)
129
+ num_cpu_blocks = int(cpu_swap_space // cache_block_size)
130
+ torch.cuda.empty_cache()
131
+ # Reset the seed to ensure that the model output is not affected by
132
+ # the profiling.
133
+ set_random_seed(self.seed)
134
+ return num_gpu_blocks, num_cpu_blocks
135
+
136
+ def init_cache_engine(self, block_size: int, num_gpu_blocks: int,
137
+ num_cpu_blocks: int):
138
+ self.block_size = block_size
139
self.cache_engine = CacheEngine(
140
worker_id=self.worker_id,
141
num_layers=self.num_layers,
142
num_heads=self.num_heads,
143
head_size=self.head_size,
74
- block_size=block_size,
144
+ block_size=self.block_size,
145
num_gpu_blocks=num_gpu_blocks,
146
num_cpu_blocks=num_cpu_blocks,
147
dtype=self.dtype,
@@ -129,6 +199,12 @@ class Worker:
199
# is always the first token in the sequence.
200
input_positions.extend(range(len(prompt_tokens)))
201
202
+ if seq_group_metadata.block_tables is None:
203
+ # During memory profiling, the block tables are not initialized
204
+ # yet. In this case, we just use a dummy slot mapping.
205
+ slot_mapping.extend([0] * prompt_len)
206
+ continue
207
+
208
# Compute the slot mapping.
209
block_table = seq_group_metadata.block_tables[seq_id]
210
for i in range(prompt_len):