@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e9d3f2ff77
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add memory analyzer & utomatically configure KV cache size (#6)

Woosuk Kwon committed Mar 11, 2023 at 23:23 UTC e9d3f2ff7772c8efe41dc805cec71c223ec18ec8
7 files changed +216 -34
README.md
+1 -1
@@ -3,7 +3,7 @@
3 ## Installation
4
5 ```bash
6 -pip install cmake torch transformers
6 +pip install psutil numpy torch transformers
7 pip install flash-attn # This may take up to 10 mins.
8 pip install -e .
9 ```
cacheflow/master/scheduler.py
+3 -3
@@ -9,8 +9,6 @@ from cacheflow.sequence import SequenceGroupInputs
9 from cacheflow.sequence import SequenceOutputs
10 from cacheflow.sequence import SequenceStatus
11
12 -_MAX_NUM_BATCHED_TOKENS = 2048
13 -
12
13 class Scheduler:
14
@@ -21,12 +19,14 @@ class Scheduler:
19 block_size: int,
20 num_gpu_blocks: int,
21 num_cpu_blocks: int,
22 + max_num_batched_tokens: int,
23 ) -> None:
24 self.frontend = frontend
25 self.controllers = controllers
26 self.block_size = block_size
27 self.num_gpu_blocks = num_gpu_blocks
28 self.num_cpu_blocks = num_cpu_blocks
29 + self.max_num_batched_tokens = max_num_batched_tokens
30
31 # Create the block space manager.
32 self.block_manager = BlockSpaceManager(
@@ -164,7 +164,7 @@ class Scheduler:
164 num_prompt_tokens = seq_group.seqs[0].get_len()
165 if self.block_manager.can_allocate(seq_group):
166 if (num_batched_tokens + num_prompt_tokens
167 - <= _MAX_NUM_BATCHED_TOKENS):
167 + <= self.max_num_batched_tokens):
168 self._allocate(seq_group)
169 num_batched_tokens += num_prompt_tokens
170 continue
cacheflow/models/__init__.py
+4 -2
@@ -1,10 +1,12 @@
1 from cacheflow.models.input_metadata import InputMetadata
2 +from cacheflow.models.model_utils import get_memory_analyzer
3 from cacheflow.models.model_utils import get_model
3 -from cacheflow.models.model_utils import set_seed
4 +from cacheflow.models.utils import set_seed
5
6
7 __all__ = [
8 'InputMetadata',
9 + 'get_memory_analyzer',
10 'get_model',
9 - 'set_seed'
11 + 'set_seed',
12 ]
cacheflow/models/memory_analyzer.py new
+125
@@ -0,0 +1,125 @@
1 +import torch
2 +from transformers import AutoConfig
3 +
4 +from cacheflow.models.utils import get_cpu_memory
5 +from cacheflow.models.utils import get_dtype_size
6 +from cacheflow.models.utils import get_gpu_memory
7 +
8 +_GiB = 1 << 30
9 +
10 +
11 +class CacheFlowMemoryAnalyzer:
12 +
13 + def get_max_num_gpu_blocks(
14 + self,
15 + max_num_batched_tokens: int,
16 + memory_utilization: float,
17 + ) -> int:
18 + raise NotImplementedError()
19 +
20 + def get_max_num_cpu_blocks(
21 + self,
22 + memory_utilization: float,
23 + ) -> int:
24 + raise NotImplementedError()
25 +
26 +
27 +class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
28 +
29 + def __init__(
30 + self,
31 + model_name: str,
32 + block_size: int,
33 + dtype: torch.dtype,
34 + ) -> None:
35 + self.model_name = model_name
36 + self.block_size = block_size
37 + self.dtype = dtype
38 +
39 + # TODO(woosuk): Support tensor parallelism.
40 + config = AutoConfig.from_pretrained(model_name)
41 + self.num_layers = config.num_hidden_layers
42 + self.hidden_size = config.hidden_size
43 + self.num_heads = config.num_attention_heads
44 + self.head_size = config.hidden_size // self.num_heads
45 + self.ffn_size = config.ffn_dim
46 + self.embedding_size = config.word_embed_proj_dim
47 + self.vocab_size = config.vocab_size
48 + self.max_position = config.max_position_embeddings
49 +
50 + def _get_param_size(self) -> int:
51 + # TODO(woosuk): Support tensor parallelism.
52 + word_embedding = self.vocab_size * self.embedding_size
53 + if self.embedding_size != self.vocab_size:
54 + # Project in/out.
55 + word_embedding += 2 * self.embedding_size * self.vocab_size
56 + position_embedding = self.max_position * self.hidden_size
57 +
58 + ln1 = 2 * self.hidden_size
59 + q = self.hidden_size * self.hidden_size + self.hidden_size
60 + k = self.hidden_size * self.hidden_size + self.hidden_size
61 + v = self.hidden_size * self.hidden_size + self.hidden_size
62 + out = self.hidden_size * self.hidden_size + self.hidden_size
63 + mha = ln1 + q + k + v + out
64 +
65 + ln2 = 2 * self.hidden_size
66 + ffn1 = self.hidden_size * self.ffn_size + self.ffn_size
67 + ffn2 = self.ffn_size * self.hidden_size + self.hidden_size
68 + ffn = ln2 + ffn1 + ffn2
69 +
70 + total = (word_embedding + position_embedding +
71 + self.num_layers * (mha + ffn))
72 + dtype_size = get_dtype_size(self.dtype)
73 + return dtype_size * total
74 +
75 + def _get_max_act_size(
76 + self,
77 + max_num_batched_tokens: int,
78 + ) -> int:
79 + # TODO(woosuk): Support tensor parallelism.
80 + # NOTE: We approxmiately calculate the maximum activation size by
81 + # 1) estimating the maximum activation tensor size during inference, and
82 + # 2) multiplying it by 4.
83 + # Here, we assume that FlashAttention is used and
84 + # thus the attention maps are never materialized in GPU DRAM.
85 + qkv = 3 * (max_num_batched_tokens * self.hidden_size)
86 + ffn = max_num_batched_tokens * self.ffn_size
87 + max_act = 4 * max(qkv, ffn)
88 + dtype_size = get_dtype_size(self.dtype)
89 + return dtype_size * max_act
90 +
91 + def _get_workspace_size(self) -> int:
92 + return 1 * _GiB
93 +
94 + def _get_cache_block_size(self) -> int:
95 + key_cache_block = self.block_size * self.num_heads * self.head_size
96 + value_cache_block = self.block_size * self.num_heads * self.head_size
97 + total = self.num_layers * (key_cache_block + value_cache_block)
98 + dtype_size = get_dtype_size(self.dtype)
99 + return dtype_size * total
100 +
101 + def get_max_num_gpu_blocks(
102 + self,
103 + max_num_batched_tokens: int,
104 + memory_utilization: float = 0.95,
105 + ) -> int:
106 + # NOTE(woosuk): This assumes that the machine has homogeneous GPUs.
107 + gpu_memory = get_gpu_memory()
108 + usable_memory = int(memory_utilization * gpu_memory)
109 +
110 + param_size = self._get_param_size()
111 + act_size = self._get_max_act_size(max_num_batched_tokens)
112 + workspace_size = self._get_workspace_size()
113 +
114 + max_cache_size = usable_memory - (param_size + act_size + workspace_size)
115 + max_num_blocks = max_cache_size // self._get_cache_block_size()
116 + return max_num_blocks
117 +
118 + def get_max_num_cpu_blocks(
119 + self,
120 + memory_utilization: float = 0.25,
121 + ) -> int:
122 + cpu_memory = get_cpu_memory()
123 + usable_memory = int(memory_utilization * cpu_memory)
124 + max_num_blocks = usable_memory // self._get_cache_block_size()
125 + return max_num_blocks
cacheflow/models/model_utils.py
+23 -21
@@ -1,21 +1,20 @@
1 -import random
1 from typing import Union
2
4 -import numpy as np
3 import torch
4 import torch.nn as nn
5
6 +from cacheflow.models.memory_analyzer import CacheFlowMemoryAnalyzer
7 +from cacheflow.models.memory_analyzer import OPTMemoryAnalyzer
8 from cacheflow.models.opt import OPTForCausalLM
9 +from cacheflow.models.utils import get_torch_dtype
10
10 -MODEL_CLASSES = {
11 +
12 +_MODELS = {
13 'opt': OPTForCausalLM,
14 }
15
14 -STR_DTYPE_TO_TORCH_DTYPE = {
15 - 'half': torch.half,
16 - 'float': torch.float,
17 - 'float16': torch.float16,
18 - 'float32': torch.float32,
16 +_MEMORY_ANALYZERS = {
17 + 'opt': OPTMemoryAnalyzer,
18 }
19
20
@@ -23,20 +22,23 @@ def get_model(
22 model_name: str,
23 dtype: Union[torch.dtype, str],
24 ) -> nn.Module:
26 - if isinstance(dtype, str):
27 - torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
28 - else:
29 - torch_dtype = dtype
30 - for model_class, hf_model in MODEL_CLASSES.items():
25 + torch_dtype = get_torch_dtype(dtype)
26 + for model_class, hf_model in _MODELS.items():
27 if model_class in model_name:
32 - model = hf_model.from_pretrained(model_name, torch_dtype=torch_dtype)
28 + model = hf_model.from_pretrained(
29 + model_name, torch_dtype=torch_dtype)
30 return model.eval()
34 - raise ValueError(f'Invalid model name: {model_name}')
31 + raise ValueError(f'Unsupported model name: {model_name}')
32
33
37 -def set_seed(seed: int) -> None:
38 - random.seed(seed)
39 - np.random.seed(seed)
40 - torch.manual_seed(seed)
41 - if torch.cuda.is_available():
42 - torch.cuda.manual_seed_all(seed)
34 +def get_memory_analyzer(
35 + model_name: str,
36 + block_size: int,
37 + dtype: Union[torch.dtype, str],
38 +) -> CacheFlowMemoryAnalyzer:
39 + torch_dtype = get_torch_dtype(dtype)
40 + for model_class, memory_analyzer in _MEMORY_ANALYZERS.items():
41 + if model_class in model_name:
42 + return memory_analyzer(
43 + model_name, block_size, torch_dtype)
44 + raise ValueError(f'Unsupported model name: {model_name}')
cacheflow/models/utils.py new
+43
@@ -0,0 +1,43 @@
1 +from typing import Union
2 +
3 +import random
4 +
5 +import numpy as np
6 +import psutil
7 +import torch
8 +
9 +_STR_DTYPE_TO_TORCH_DTYPE = {
10 + 'half': torch.half,
11 + 'float': torch.float,
12 + 'float16': torch.float16,
13 + 'float32': torch.float32,
14 +}
15 +
16 +
17 +def get_torch_dtype(dtype: Union[torch.dtype, str]) -> torch.dtype:
18 + if isinstance(dtype, str):
19 + torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
20 + else:
21 + torch_dtype = dtype
22 + return torch_dtype
23 +
24 +
25 +def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
26 + torch_dtype = get_torch_dtype(dtype)
27 + return torch.tensor([], dtype=torch_dtype).element_size()
28 +
29 +
30 +def set_seed(seed: int) -> None:
31 + random.seed(seed)
32 + np.random.seed(seed)
33 + torch.manual_seed(seed)
34 + if torch.cuda.is_available():
35 + torch.cuda.manual_seed_all(seed)
36 +
37 +
38 +def get_gpu_memory(gpu: int = 0) -> int:
39 + return torch.cuda.get_device_properties(gpu).total_memory
40 +
41 +
42 +def get_cpu_memory() -> int:
43 + return psutil.virtual_memory().total
server.py
+17 -7
@@ -3,6 +3,7 @@ from typing import List
3
4 from cacheflow.master.frontend import Frontend
5 from cacheflow.master.scheduler import Scheduler
6 +from cacheflow.models import get_memory_analyzer
7 from cacheflow.worker.controller import Controller
8
9 parser = argparse.ArgumentParser(description='CacheFlow server')
@@ -10,17 +11,25 @@ parser.add_argument('--model', type=str, default='facebook/opt-125m', help='mode
11 parser.add_argument('--num-nodes', type=int, default=1, help='number of nodes')
12 parser.add_argument('--num-workers', type=int, default=1, help='number of workers per node')
13 parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='token block size')
13 -# TODO(woosuk): Add an analytical model to determine the maximum number of GPU/CPU blocks.
14 -parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of GPU blocks (per GPU)')
15 -parser.add_argument('--num-cpu-blocks', type=int, default=32, help='number of CPU blocks (per GPU)')
14 # NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
15 parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
16 # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
17 parser.add_argument('--seed', type=int, default=0, help='random seed')
18 +parser.add_argument('--max-batch-size', type=int, default=2048, help='maximum number of batched tokens')
19 args = parser.parse_args()
20
21
22 def main():
23 + memory_analyzer = get_memory_analyzer(
24 + model_name=args.model,
25 + block_size=args.block_size,
26 + dtype=args.dtype,
27 + )
28 + num_gpu_blocks = memory_analyzer.get_max_num_gpu_blocks(
29 + max_num_batched_tokens=args.max_batch_size)
30 + num_cpu_blocks = memory_analyzer.get_max_num_cpu_blocks()
31 + print(f'# GPU blocks: {num_gpu_blocks}, # CPU blocks: {num_cpu_blocks}')
32 +
33 # Create a controller for each node.
34 controllers: List[Controller] = []
35 for i in range(args.num_nodes):
@@ -29,8 +38,8 @@ def main():
38 num_workers=args.num_workers,
39 model_name=args.model,
40 block_size=args.block_size,
32 - num_gpu_blocks=args.num_gpu_blocks,
33 - num_cpu_blocks=args.num_cpu_blocks,
41 + num_gpu_blocks=num_gpu_blocks,
42 + num_cpu_blocks=num_cpu_blocks,
43 dtype=args.dtype,
44 seed=args.seed,
45 )
@@ -47,8 +56,9 @@ def main():
56 frontend=frontend,
57 controllers=controllers,
58 block_size=args.block_size,
50 - num_gpu_blocks=args.num_gpu_blocks,
51 - num_cpu_blocks=args.num_cpu_blocks,
59 + num_gpu_blocks=num_gpu_blocks,
60 + num_cpu_blocks=num_cpu_blocks,
61 + max_num_batched_tokens=args.max_batch_size,
62 )
63 # Connect the controllers.
64 for i in range(len(controllers) - 1):