A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add CUDA graph-based all reduce launcher (#26)
Woosuk Kwon committed
Apr 5, 2023 at 11:16 UTC
12659a0bd7e442d1e3dcbff4a4d5909a02c76c60
7 files changed
+103
-16
benchmark/benchmark_latency.py
+3
-2
@@ -35,7 +35,7 @@ def main(args: argparse.Namespace):
35
dtype=args.dtype,
36
seed=args.seed,
37
swap_space=args.swap_space,
38
- max_batch_size=args.max_batch_size,
38
+ max_num_batched_tokens=args.max_num_batched_tokens,
39
num_nodes=num_nodes,
40
num_devices_per_node=num_devices_per_node,
41
distributed_init_method=distributed_init_method,
@@ -94,6 +94,7 @@ if __name__ == '__main__':
94
parser.add_argument('--output-len', type=int, default=128)
95
parser.add_argument('--batch-size', type=int, default=8)
96
args = parser.parse_args()
97
- args.max_batch_size = max(args.max_batch_size, args.batch_size * args.input_len)
97
+ args.max_num_batched_tokens = max(
98
+ args.max_num_batched_tokens, args.batch_size * args.input_len)
99
print(args)
100
main(args)
cacheflow/master/server.py
+7
-6
@@ -22,7 +22,7 @@ class Server:
22
dtype: str,
23
seed: int,
24
swap_space: int,
25
- max_batch_size: int,
25
+ max_num_batched_tokens: int,
26
num_nodes: int,
27
num_devices_per_node: int,
28
distributed_init_method: str,
@@ -43,7 +43,7 @@ class Server:
43
tensor_parallel_size=tensor_parallel_size,
44
)
45
self.num_gpu_blocks = self.memory_analyzer.get_max_num_gpu_blocks(
46
- max_num_batched_tokens=max_batch_size)
46
+ max_num_batched_tokens=max_num_batched_tokens)
47
self.num_cpu_blocks = self.memory_analyzer.get_max_num_cpu_blocks(
48
swap_space=swap_space)
49
print(f'# GPU blocks: {self.num_gpu_blocks}, '
@@ -66,6 +66,7 @@ class Server:
66
dtype=dtype,
67
seed=seed,
68
model_path=model_path,
69
+ max_num_batched_tokens=max_num_batched_tokens,
70
)
71
self.controllers.append(controller)
72
@@ -75,7 +76,7 @@ class Server:
76
block_size=block_size,
77
num_gpu_blocks=self.num_gpu_blocks,
78
num_cpu_blocks=self.num_cpu_blocks,
78
- max_num_batched_tokens=max_batch_size,
79
+ max_num_batched_tokens=max_num_batched_tokens,
80
)
81
# Connect the controllers.
82
for i in range(len(self.controllers) - 1):
@@ -168,8 +169,8 @@ def add_server_arguments(parser: argparse.ArgumentParser):
169
parser.add_argument('--model-path', type=str, default='~/.cacheflow/model_weights',
170
help='model path to download and load the weights')
171
# Parallel arguments
171
- parser.add_argument('--pipeline-parallel-size', type=int, default=1, help='number of pipeline stages')
172
- parser.add_argument('--tensor-parallel-size', type=int, default=1, help='number of tensor parallel replicas')
172
+ parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
173
+ parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
174
# KV cache arguments
175
parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='token block size')
176
# NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
@@ -177,5 +178,5 @@ def add_server_arguments(parser: argparse.ArgumentParser):
178
# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
179
parser.add_argument('--seed', type=int, default=0, help='random seed')
180
parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
180
- parser.add_argument('--max-batch-size', type=int, default=2560, help='maximum number of batched tokens')
181
+ parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens')
182
return parser
cacheflow/parallel_utils/parallel_state.py
+71
@@ -47,6 +47,7 @@ _DATA_PARALLEL_GLOBAL_RANKS = None
47
# Memory buffers to avoid dynamic memory allocation
48
_GLOBAL_MEMORY_BUFFER = None
49
50
+_ALL_REDUCE_LAUNCHER: Optional['GraphAllReduce'] = None
51
52
def initialize_model_parallel(
53
tensor_model_parallel_size: int = 1,
@@ -205,6 +206,20 @@ def initialize_model_parallel(
206
_set_global_memory_buffer()
207
208
209
+def initialize_all_reduce_launcher(
210
+ max_num_tokens: int,
211
+ hidden_size: int,
212
+ dtype: torch.dtype,
213
+ disable_graph: bool = False,
214
+) -> None:
215
+ global _ALL_REDUCE_LAUNCHER
216
+ _ALL_REDUCE_LAUNCHER = GraphAllReduce(
217
+ max_num_tokens=max_num_tokens,
218
+ hidden_size=hidden_size,
219
+ dtype=dtype,
220
+ disable_graph=disable_graph,
221
+ )
222
+
223
def model_parallel_is_initialized():
224
"""Check if model and data parallel groups are initialized."""
225
if _TENSOR_MODEL_PARALLEL_GROUP is None or \
@@ -491,6 +506,9 @@ def get_global_memory_buffer():
506
assert _GLOBAL_MEMORY_BUFFER is not None, 'global memory buffer is not initialized'
507
return _GLOBAL_MEMORY_BUFFER
508
509
+def get_all_reduce_launcher() -> 'GraphAllReduce':
510
+ assert _ALL_REDUCE_LAUNCHER is not None, 'all reduce launcher is not initialized'
511
+ return _ALL_REDUCE_LAUNCHER
512
513
def destroy_model_parallel():
514
"""Set the groups to none."""
@@ -520,3 +538,56 @@ def destroy_model_parallel():
538
_MPU_PIPELINE_MODEL_PARALLEL_RANK = None
539
global _GLOBAL_MEMORY_BUFFER
540
_GLOBAL_MEMORY_BUFFER = None
541
+
542
+
543
+class GraphAllReduce:
544
+
545
+ def __init__(
546
+ self,
547
+ max_num_tokens: int,
548
+ hidden_size: int,
549
+ dtype: torch.dtype,
550
+ disable_graph: bool = False,
551
+ ) -> None:
552
+ self.max_num_tokens = max_num_tokens
553
+ self.hidden_size = hidden_size
554
+ self.disable_graph = disable_graph
555
+
556
+ tp_world_size = get_tensor_model_parallel_world_size()
557
+ if tp_world_size == 1:
558
+ return
559
+
560
+ self.group = get_tensor_model_parallel_group()
561
+ self.buffer = torch.empty(
562
+ size=(max_num_tokens, hidden_size),
563
+ dtype=dtype,
564
+ device='cuda',
565
+ )
566
+
567
+ # Build graphs for different number of tokens.
568
+ if not self.disable_graph:
569
+ self.graphs = {}
570
+ for num_tokens in range(8, max_num_tokens + 1, 8):
571
+ self.graphs[num_tokens] = self._build_graph(num_tokens)
572
+
573
+ def _build_graph(self, num_tokens: int) -> torch.cuda.CUDAGraph:
574
+ # Warm up.
575
+ torch.distributed.all_reduce(self.buffer[:num_tokens], group=self.group)
576
+ torch.cuda.synchronize()
577
+
578
+ # Build graph.
579
+ graph = torch.cuda.CUDAGraph()
580
+ with torch.cuda.graph(graph):
581
+ torch.distributed.all_reduce(
582
+ self.buffer[:num_tokens], group=self.group)
583
+ torch.cuda.synchronize()
584
+ return graph
585
+
586
+ def launch(self, x: torch.Tensor) -> torch.Tensor:
587
+ # NOTE: x must be a slice of self.buffer.
588
+ num_tokens = x.shape[0]
589
+ if self.disable_graph:
590
+ torch.distributed.all_reduce(x, group=self.group)
591
+ else:
592
+ self.graphs[num_tokens].replay()
593
+ return x
cacheflow/parallel_utils/tensor_parallel/layers.py
+13
-6
@@ -12,6 +12,7 @@ from torch.nn.parameter import Parameter
12
from cacheflow.parallel_utils.parallel_state import (
13
get_tensor_model_parallel_rank,
14
get_tensor_model_parallel_world_size,
15
+ get_all_reduce_launcher,
16
)
17
from .mappings import (
18
copy_to_tensor_model_parallel_region,
@@ -407,8 +408,7 @@ class RowParallelLinear(torch.nn.Module):
408
self.bias.zero_()
409
else:
410
self.register_parameter('bias', None)
410
-
411
-
411
+ self.weight_t = self.weight.t()
412
413
def forward(self, input_):
414
"""Forward of RowParallelLinear
@@ -425,11 +425,18 @@ class RowParallelLinear(torch.nn.Module):
425
input_parallel = input_
426
else:
427
input_parallel = scatter_to_tensor_model_parallel_region(input_)
428
- # Matrix multiply.
429
- output_parallel = F.linear(input_parallel, self.weight)
428
+ if get_tensor_model_parallel_world_size() == 1:
429
+ # Matrix multiply.
430
+ output_ = F.linear(input_parallel, self.weight)
431
+ else:
432
+ # Matrix multiply.
433
+ all_reduce_launcher = get_all_reduce_launcher()
434
+ num_tokens = input_parallel.shape[0]
435
+ output_buffer = all_reduce_launcher.buffer[:num_tokens]
436
+ torch.matmul(input_parallel, self.weight_t, out=output_buffer)
437
+ # All-reduce across all the partitions.
438
+ output_ = all_reduce_launcher.launch(output_buffer)
439
431
- # All-reduce across all the partitions.
432
- output_ = reduce_from_tensor_model_parallel_region(output_parallel)
440
if not self.skip_bias_add:
441
output = output_ + self.bias if self.bias is not None else output_
442
output_bias = None
cacheflow/worker/controller.py
+2
@@ -27,6 +27,7 @@ class Controller:
27
dtype: str,
28
seed: int,
29
model_path: str,
30
+ max_num_batched_tokens: int,
31
) -> None:
32
self.stage_id = stage_id
33
self.stage_devices = stage_devices
@@ -57,6 +58,7 @@ class Controller:
58
tensor_parallel_size=tensor_parallel_size,
59
pipeline_parallel_size=pipeline_parallel_size,
60
model_path=model_path,
61
+ max_num_batched_tokens=max_num_batched_tokens,
62
)
63
self.workers.append(worker)
64
cacheflow/worker/worker.py
+6
-1
@@ -9,7 +9,9 @@ from cacheflow.sequence import SequenceGroupInputs
9
from cacheflow.sequence import SequenceOutputs
10
from cacheflow.worker.cache_engine import CacheEngine
11
from cacheflow.parallel_utils.parallel_state import (
12
- initialize_model_parallel, get_tensor_model_parallel_world_size)
12
+ initialize_model_parallel,
13
+ initialize_all_reduce_launcher,
14
+ get_tensor_model_parallel_world_size)
15
from cacheflow.utils import set_random_seed
16
17
@@ -27,6 +29,7 @@ class Worker:
29
rank: int,
30
world_size: int,
31
model_path: str,
32
+ max_num_batched_tokens: int,
33
tensor_parallel_size: int = 1,
34
pipeline_parallel_size: int = 1,
35
) -> None:
@@ -44,6 +47,8 @@ class Worker:
47
self.model = self.model.cuda()
48
tensor_model_parallel_world_size = (
49
get_tensor_model_parallel_world_size())
50
+ initialize_all_reduce_launcher(
51
+ max_num_batched_tokens, self.model.config.hidden_size, self.dtype)
52
self.num_layers = self.model.config.num_hidden_layers
53
assert self.model.config.num_attention_heads % tensor_model_parallel_world_size == 0
54
self.num_heads = self.model.config.num_attention_heads // tensor_model_parallel_world_size
simple_server.py
+1
-1
@@ -28,7 +28,7 @@ def main(args: argparse.Namespace):
28
dtype=args.dtype,
29
seed=args.seed,
30
swap_space=args.swap_space,
31
- max_batch_size=args.max_batch_size,
31
+ max_num_batched_tokens=args.max_num_batched_tokens,
32
num_nodes=num_nodes,
33
num_devices_per_node=num_devices_per_node,
34
distributed_init_method=distributed_init_method,