A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add memory analyzer & utomatically configure KV cache size (#6)
Woosuk Kwon committed
Mar 11, 2023 at 23:23 UTC
e9d3f2ff7772c8efe41dc805cec71c223ec18ec8
7 files changed
+216
-34
README.md
+1
-1
@@ -3,7 +3,7 @@
3
## Installation
4
5
```bash
6
-pip install cmake torch transformers
6
+pip install psutil numpy torch transformers
7
pip install flash-attn # This may take up to 10 mins.
8
pip install -e .
9
```
cacheflow/master/scheduler.py
+3
-3
@@ -9,8 +9,6 @@ from cacheflow.sequence import SequenceGroupInputs
9
from cacheflow.sequence import SequenceOutputs
10
from cacheflow.sequence import SequenceStatus
11
12
-_MAX_NUM_BATCHED_TOKENS = 2048
13
-
12
13
class Scheduler:
14
@@ -21,12 +19,14 @@ class Scheduler:
19
block_size: int,
20
num_gpu_blocks: int,
21
num_cpu_blocks: int,
22
+ max_num_batched_tokens: int,
23
) -> None:
24
self.frontend = frontend
25
self.controllers = controllers
26
self.block_size = block_size
27
self.num_gpu_blocks = num_gpu_blocks
28
self.num_cpu_blocks = num_cpu_blocks
29
+ self.max_num_batched_tokens = max_num_batched_tokens
30
31
# Create the block space manager.
32
self.block_manager = BlockSpaceManager(
@@ -164,7 +164,7 @@ class Scheduler:
164
num_prompt_tokens = seq_group.seqs[0].get_len()
165
if self.block_manager.can_allocate(seq_group):
166
if (num_batched_tokens + num_prompt_tokens
167
- <= _MAX_NUM_BATCHED_TOKENS):
167
+ <= self.max_num_batched_tokens):
168
self._allocate(seq_group)
169
num_batched_tokens += num_prompt_tokens
170
continue
cacheflow/models/__init__.py
+4
-2
@@ -1,10 +1,12 @@
1
from cacheflow.models.input_metadata import InputMetadata
2
+from cacheflow.models.model_utils import get_memory_analyzer
3
from cacheflow.models.model_utils import get_model
3
-from cacheflow.models.model_utils import set_seed
4
+from cacheflow.models.utils import set_seed
5
6
7
__all__ = [
8
'InputMetadata',
9
+ 'get_memory_analyzer',
10
'get_model',
9
- 'set_seed'
11
+ 'set_seed',
12
]
cacheflow/models/memory_analyzer.py
new
+125
@@ -0,0 +1,125 @@
1
+import torch
2
+from transformers import AutoConfig
3
+
4
+from cacheflow.models.utils import get_cpu_memory
5
+from cacheflow.models.utils import get_dtype_size
6
+from cacheflow.models.utils import get_gpu_memory
7
+
8
+_GiB = 1 << 30
9
+
10
+
11
+class CacheFlowMemoryAnalyzer:
12
+
13
+ def get_max_num_gpu_blocks(
14
+ self,
15
+ max_num_batched_tokens: int,
16
+ memory_utilization: float,
17
+ ) -> int:
18
+ raise NotImplementedError()
19
+
20
+ def get_max_num_cpu_blocks(
21
+ self,
22
+ memory_utilization: float,
23
+ ) -> int:
24
+ raise NotImplementedError()
25
+
26
+
27
+class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
28
+
29
+ def __init__(
30
+ self,
31
+ model_name: str,
32
+ block_size: int,
33
+ dtype: torch.dtype,
34
+ ) -> None:
35
+ self.model_name = model_name
36
+ self.block_size = block_size
37
+ self.dtype = dtype
38
+
39
+ # TODO(woosuk): Support tensor parallelism.
40
+ config = AutoConfig.from_pretrained(model_name)
41
+ self.num_layers = config.num_hidden_layers
42
+ self.hidden_size = config.hidden_size
43
+ self.num_heads = config.num_attention_heads
44
+ self.head_size = config.hidden_size // self.num_heads
45
+ self.ffn_size = config.ffn_dim
46
+ self.embedding_size = config.word_embed_proj_dim
47
+ self.vocab_size = config.vocab_size
48
+ self.max_position = config.max_position_embeddings
49
+
50
+ def _get_param_size(self) -> int:
51
+ # TODO(woosuk): Support tensor parallelism.
52
+ word_embedding = self.vocab_size * self.embedding_size
53
+ if self.embedding_size != self.vocab_size:
54
+ # Project in/out.
55
+ word_embedding += 2 * self.embedding_size * self.vocab_size
56
+ position_embedding = self.max_position * self.hidden_size
57
+
58
+ ln1 = 2 * self.hidden_size
59
+ q = self.hidden_size * self.hidden_size + self.hidden_size
60
+ k = self.hidden_size * self.hidden_size + self.hidden_size
61
+ v = self.hidden_size * self.hidden_size + self.hidden_size
62
+ out = self.hidden_size * self.hidden_size + self.hidden_size
63
+ mha = ln1 + q + k + v + out
64
+
65
+ ln2 = 2 * self.hidden_size
66
+ ffn1 = self.hidden_size * self.ffn_size + self.ffn_size
67
+ ffn2 = self.ffn_size * self.hidden_size + self.hidden_size
68
+ ffn = ln2 + ffn1 + ffn2
69
+
70
+ total = (word_embedding + position_embedding +
71
+ self.num_layers * (mha + ffn))
72
+ dtype_size = get_dtype_size(self.dtype)
73
+ return dtype_size * total
74
+
75
+ def _get_max_act_size(
76
+ self,
77
+ max_num_batched_tokens: int,
78
+ ) -> int:
79
+ # TODO(woosuk): Support tensor parallelism.
80
+ # NOTE: We approxmiately calculate the maximum activation size by
81
+ # 1) estimating the maximum activation tensor size during inference, and
82
+ # 2) multiplying it by 4.
83
+ # Here, we assume that FlashAttention is used and
84
+ # thus the attention maps are never materialized in GPU DRAM.
85
+ qkv = 3 * (max_num_batched_tokens * self.hidden_size)
86
+ ffn = max_num_batched_tokens * self.ffn_size
87
+ max_act = 4 * max(qkv, ffn)
88
+ dtype_size = get_dtype_size(self.dtype)
89
+ return dtype_size * max_act
90
+
91
+ def _get_workspace_size(self) -> int:
92
+ return 1 * _GiB
93
+
94
+ def _get_cache_block_size(self) -> int:
95
+ key_cache_block = self.block_size * self.num_heads * self.head_size
96
+ value_cache_block = self.block_size * self.num_heads * self.head_size
97
+ total = self.num_layers * (key_cache_block + value_cache_block)
98
+ dtype_size = get_dtype_size(self.dtype)
99
+ return dtype_size * total
100
+
101
+ def get_max_num_gpu_blocks(
102
+ self,
103
+ max_num_batched_tokens: int,
104
+ memory_utilization: float = 0.95,
105
+ ) -> int:
106
+ # NOTE(woosuk): This assumes that the machine has homogeneous GPUs.
107
+ gpu_memory = get_gpu_memory()
108
+ usable_memory = int(memory_utilization * gpu_memory)
109
+
110
+ param_size = self._get_param_size()
111
+ act_size = self._get_max_act_size(max_num_batched_tokens)
112
+ workspace_size = self._get_workspace_size()
113
+
114
+ max_cache_size = usable_memory - (param_size + act_size + workspace_size)
115
+ max_num_blocks = max_cache_size // self._get_cache_block_size()
116
+ return max_num_blocks
117
+
118
+ def get_max_num_cpu_blocks(
119
+ self,
120
+ memory_utilization: float = 0.25,
121
+ ) -> int:
122
+ cpu_memory = get_cpu_memory()
123
+ usable_memory = int(memory_utilization * cpu_memory)
124
+ max_num_blocks = usable_memory // self._get_cache_block_size()
125
+ return max_num_blocks
cacheflow/models/model_utils.py
+23
-21
@@ -1,21 +1,20 @@
1
-import random
1
from typing import Union
2
4
-import numpy as np
3
import torch
4
import torch.nn as nn
5
6
+from cacheflow.models.memory_analyzer import CacheFlowMemoryAnalyzer
7
+from cacheflow.models.memory_analyzer import OPTMemoryAnalyzer
8
from cacheflow.models.opt import OPTForCausalLM
9
+from cacheflow.models.utils import get_torch_dtype
10
10
-MODEL_CLASSES = {
11
+
12
+_MODELS = {
13
'opt': OPTForCausalLM,
14
}
15
14
-STR_DTYPE_TO_TORCH_DTYPE = {
15
- 'half': torch.half,
16
- 'float': torch.float,
17
- 'float16': torch.float16,
18
- 'float32': torch.float32,
16
+_MEMORY_ANALYZERS = {
17
+ 'opt': OPTMemoryAnalyzer,
18
}
19
20
@@ -23,20 +22,23 @@ def get_model(
22
model_name: str,
23
dtype: Union[torch.dtype, str],
24
) -> nn.Module:
26
- if isinstance(dtype, str):
27
- torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
28
- else:
29
- torch_dtype = dtype
30
- for model_class, hf_model in MODEL_CLASSES.items():
25
+ torch_dtype = get_torch_dtype(dtype)
26
+ for model_class, hf_model in _MODELS.items():
27
if model_class in model_name:
32
- model = hf_model.from_pretrained(model_name, torch_dtype=torch_dtype)
28
+ model = hf_model.from_pretrained(
29
+ model_name, torch_dtype=torch_dtype)
30
return model.eval()
34
- raise ValueError(f'Invalid model name: {model_name}')
31
+ raise ValueError(f'Unsupported model name: {model_name}')
32
33
37
-def set_seed(seed: int) -> None:
38
- random.seed(seed)
39
- np.random.seed(seed)
40
- torch.manual_seed(seed)
41
- if torch.cuda.is_available():
42
- torch.cuda.manual_seed_all(seed)
34
+def get_memory_analyzer(
35
+ model_name: str,
36
+ block_size: int,
37
+ dtype: Union[torch.dtype, str],
38
+) -> CacheFlowMemoryAnalyzer:
39
+ torch_dtype = get_torch_dtype(dtype)
40
+ for model_class, memory_analyzer in _MEMORY_ANALYZERS.items():
41
+ if model_class in model_name:
42
+ return memory_analyzer(
43
+ model_name, block_size, torch_dtype)
44
+ raise ValueError(f'Unsupported model name: {model_name}')
cacheflow/models/utils.py
new
+43
@@ -0,0 +1,43 @@
1
+from typing import Union
2
+
3
+import random
4
+
5
+import numpy as np
6
+import psutil
7
+import torch
8
+
9
+_STR_DTYPE_TO_TORCH_DTYPE = {
10
+ 'half': torch.half,
11
+ 'float': torch.float,
12
+ 'float16': torch.float16,
13
+ 'float32': torch.float32,
14
+}
15
+
16
+
17
+def get_torch_dtype(dtype: Union[torch.dtype, str]) -> torch.dtype:
18
+ if isinstance(dtype, str):
19
+ torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
20
+ else:
21
+ torch_dtype = dtype
22
+ return torch_dtype
23
+
24
+
25
+def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
26
+ torch_dtype = get_torch_dtype(dtype)
27
+ return torch.tensor([], dtype=torch_dtype).element_size()
28
+
29
+
30
+def set_seed(seed: int) -> None:
31
+ random.seed(seed)
32
+ np.random.seed(seed)
33
+ torch.manual_seed(seed)
34
+ if torch.cuda.is_available():
35
+ torch.cuda.manual_seed_all(seed)
36
+
37
+
38
+def get_gpu_memory(gpu: int = 0) -> int:
39
+ return torch.cuda.get_device_properties(gpu).total_memory
40
+
41
+
42
+def get_cpu_memory() -> int:
43
+ return psutil.virtual_memory().total
server.py
+17
-7
@@ -3,6 +3,7 @@ from typing import List
3
4
from cacheflow.master.frontend import Frontend
5
from cacheflow.master.scheduler import Scheduler
6
+from cacheflow.models import get_memory_analyzer
7
from cacheflow.worker.controller import Controller
8
9
parser = argparse.ArgumentParser(description='CacheFlow server')
@@ -10,17 +11,25 @@ parser.add_argument('--model', type=str, default='facebook/opt-125m', help='mode
11
parser.add_argument('--num-nodes', type=int, default=1, help='number of nodes')
12
parser.add_argument('--num-workers', type=int, default=1, help='number of workers per node')
13
parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='token block size')
13
-# TODO(woosuk): Add an analytical model to determine the maximum number of GPU/CPU blocks.
14
-parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of GPU blocks (per GPU)')
15
-parser.add_argument('--num-cpu-blocks', type=int, default=32, help='number of CPU blocks (per GPU)')
14
# NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
15
parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
16
# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
17
parser.add_argument('--seed', type=int, default=0, help='random seed')
18
+parser.add_argument('--max-batch-size', type=int, default=2048, help='maximum number of batched tokens')
19
args = parser.parse_args()
20
21
22
def main():
23
+ memory_analyzer = get_memory_analyzer(
24
+ model_name=args.model,
25
+ block_size=args.block_size,
26
+ dtype=args.dtype,
27
+ )
28
+ num_gpu_blocks = memory_analyzer.get_max_num_gpu_blocks(
29
+ max_num_batched_tokens=args.max_batch_size)
30
+ num_cpu_blocks = memory_analyzer.get_max_num_cpu_blocks()
31
+ print(f'# GPU blocks: {num_gpu_blocks}, # CPU blocks: {num_cpu_blocks}')
32
+
33
# Create a controller for each node.
34
controllers: List[Controller] = []
35
for i in range(args.num_nodes):
@@ -29,8 +38,8 @@ def main():
38
num_workers=args.num_workers,
39
model_name=args.model,
40
block_size=args.block_size,
32
- num_gpu_blocks=args.num_gpu_blocks,
33
- num_cpu_blocks=args.num_cpu_blocks,
41
+ num_gpu_blocks=num_gpu_blocks,
42
+ num_cpu_blocks=num_cpu_blocks,
43
dtype=args.dtype,
44
seed=args.seed,
45
)
@@ -47,8 +56,9 @@ def main():
56
frontend=frontend,
57
controllers=controllers,
58
block_size=args.block_size,
50
- num_gpu_blocks=args.num_gpu_blocks,
51
- num_cpu_blocks=args.num_cpu_blocks,
59
+ num_gpu_blocks=num_gpu_blocks,
60
+ num_cpu_blocks=num_cpu_blocks,
61
+ max_num_batched_tokens=args.max_batch_size,
62
)
63
# Connect the controllers.
64
for i in range(len(controllers) - 1):