@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 12659a0bd7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add CUDA graph-based all reduce launcher (#26)

Woosuk Kwon committed Apr 5, 2023 at 11:16 UTC 12659a0bd7e442d1e3dcbff4a4d5909a02c76c60
7 files changed +103 -16
benchmark/benchmark_latency.py
+3 -2
@@ -35,7 +35,7 @@ def main(args: argparse.Namespace):
35 dtype=args.dtype,
36 seed=args.seed,
37 swap_space=args.swap_space,
38 - max_batch_size=args.max_batch_size,
38 + max_num_batched_tokens=args.max_num_batched_tokens,
39 num_nodes=num_nodes,
40 num_devices_per_node=num_devices_per_node,
41 distributed_init_method=distributed_init_method,
@@ -94,6 +94,7 @@ if __name__ == '__main__':
94 parser.add_argument('--output-len', type=int, default=128)
95 parser.add_argument('--batch-size', type=int, default=8)
96 args = parser.parse_args()
97 - args.max_batch_size = max(args.max_batch_size, args.batch_size * args.input_len)
97 + args.max_num_batched_tokens = max(
98 + args.max_num_batched_tokens, args.batch_size * args.input_len)
99 print(args)
100 main(args)
cacheflow/master/server.py
+7 -6
@@ -22,7 +22,7 @@ class Server:
22 dtype: str,
23 seed: int,
24 swap_space: int,
25 - max_batch_size: int,
25 + max_num_batched_tokens: int,
26 num_nodes: int,
27 num_devices_per_node: int,
28 distributed_init_method: str,
@@ -43,7 +43,7 @@ class Server:
43 tensor_parallel_size=tensor_parallel_size,
44 )
45 self.num_gpu_blocks = self.memory_analyzer.get_max_num_gpu_blocks(
46 - max_num_batched_tokens=max_batch_size)
46 + max_num_batched_tokens=max_num_batched_tokens)
47 self.num_cpu_blocks = self.memory_analyzer.get_max_num_cpu_blocks(
48 swap_space=swap_space)
49 print(f'# GPU blocks: {self.num_gpu_blocks}, '
@@ -66,6 +66,7 @@ class Server:
66 dtype=dtype,
67 seed=seed,
68 model_path=model_path,
69 + max_num_batched_tokens=max_num_batched_tokens,
70 )
71 self.controllers.append(controller)
72
@@ -75,7 +76,7 @@ class Server:
76 block_size=block_size,
77 num_gpu_blocks=self.num_gpu_blocks,
78 num_cpu_blocks=self.num_cpu_blocks,
78 - max_num_batched_tokens=max_batch_size,
79 + max_num_batched_tokens=max_num_batched_tokens,
80 )
81 # Connect the controllers.
82 for i in range(len(self.controllers) - 1):
@@ -168,8 +169,8 @@ def add_server_arguments(parser: argparse.ArgumentParser):
169 parser.add_argument('--model-path', type=str, default='~/.cacheflow/model_weights',
170 help='model path to download and load the weights')
171 # Parallel arguments
171 - parser.add_argument('--pipeline-parallel-size', type=int, default=1, help='number of pipeline stages')
172 - parser.add_argument('--tensor-parallel-size', type=int, default=1, help='number of tensor parallel replicas')
172 + parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
173 + parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
174 # KV cache arguments
175 parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='token block size')
176 # NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
@@ -177,5 +178,5 @@ def add_server_arguments(parser: argparse.ArgumentParser):
178 # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
179 parser.add_argument('--seed', type=int, default=0, help='random seed')
180 parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
180 - parser.add_argument('--max-batch-size', type=int, default=2560, help='maximum number of batched tokens')
181 + parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens')
182 return parser
cacheflow/parallel_utils/parallel_state.py
+71
@@ -47,6 +47,7 @@ _DATA_PARALLEL_GLOBAL_RANKS = None
47 # Memory buffers to avoid dynamic memory allocation
48 _GLOBAL_MEMORY_BUFFER = None
49
50 +_ALL_REDUCE_LAUNCHER: Optional['GraphAllReduce'] = None
51
52 def initialize_model_parallel(
53 tensor_model_parallel_size: int = 1,
@@ -205,6 +206,20 @@ def initialize_model_parallel(
206 _set_global_memory_buffer()
207
208
209 +def initialize_all_reduce_launcher(
210 + max_num_tokens: int,
211 + hidden_size: int,
212 + dtype: torch.dtype,
213 + disable_graph: bool = False,
214 +) -> None:
215 + global _ALL_REDUCE_LAUNCHER
216 + _ALL_REDUCE_LAUNCHER = GraphAllReduce(
217 + max_num_tokens=max_num_tokens,
218 + hidden_size=hidden_size,
219 + dtype=dtype,
220 + disable_graph=disable_graph,
221 + )
222 +
223 def model_parallel_is_initialized():
224 """Check if model and data parallel groups are initialized."""
225 if _TENSOR_MODEL_PARALLEL_GROUP is None or \
@@ -491,6 +506,9 @@ def get_global_memory_buffer():
506 assert _GLOBAL_MEMORY_BUFFER is not None, 'global memory buffer is not initialized'
507 return _GLOBAL_MEMORY_BUFFER
508
509 +def get_all_reduce_launcher() -> 'GraphAllReduce':
510 + assert _ALL_REDUCE_LAUNCHER is not None, 'all reduce launcher is not initialized'
511 + return _ALL_REDUCE_LAUNCHER
512
513 def destroy_model_parallel():
514 """Set the groups to none."""
@@ -520,3 +538,56 @@ def destroy_model_parallel():
538 _MPU_PIPELINE_MODEL_PARALLEL_RANK = None
539 global _GLOBAL_MEMORY_BUFFER
540 _GLOBAL_MEMORY_BUFFER = None
541 +
542 +
543 +class GraphAllReduce:
544 +
545 + def __init__(
546 + self,
547 + max_num_tokens: int,
548 + hidden_size: int,
549 + dtype: torch.dtype,
550 + disable_graph: bool = False,
551 + ) -> None:
552 + self.max_num_tokens = max_num_tokens
553 + self.hidden_size = hidden_size
554 + self.disable_graph = disable_graph
555 +
556 + tp_world_size = get_tensor_model_parallel_world_size()
557 + if tp_world_size == 1:
558 + return
559 +
560 + self.group = get_tensor_model_parallel_group()
561 + self.buffer = torch.empty(
562 + size=(max_num_tokens, hidden_size),
563 + dtype=dtype,
564 + device='cuda',
565 + )
566 +
567 + # Build graphs for different number of tokens.
568 + if not self.disable_graph:
569 + self.graphs = {}
570 + for num_tokens in range(8, max_num_tokens + 1, 8):
571 + self.graphs[num_tokens] = self._build_graph(num_tokens)
572 +
573 + def _build_graph(self, num_tokens: int) -> torch.cuda.CUDAGraph:
574 + # Warm up.
575 + torch.distributed.all_reduce(self.buffer[:num_tokens], group=self.group)
576 + torch.cuda.synchronize()
577 +
578 + # Build graph.
579 + graph = torch.cuda.CUDAGraph()
580 + with torch.cuda.graph(graph):
581 + torch.distributed.all_reduce(
582 + self.buffer[:num_tokens], group=self.group)
583 + torch.cuda.synchronize()
584 + return graph
585 +
586 + def launch(self, x: torch.Tensor) -> torch.Tensor:
587 + # NOTE: x must be a slice of self.buffer.
588 + num_tokens = x.shape[0]
589 + if self.disable_graph:
590 + torch.distributed.all_reduce(x, group=self.group)
591 + else:
592 + self.graphs[num_tokens].replay()
593 + return x
cacheflow/parallel_utils/tensor_parallel/layers.py
+13 -6
@@ -12,6 +12,7 @@ from torch.nn.parameter import Parameter
12 from cacheflow.parallel_utils.parallel_state import (
13 get_tensor_model_parallel_rank,
14 get_tensor_model_parallel_world_size,
15 + get_all_reduce_launcher,
16 )
17 from .mappings import (
18 copy_to_tensor_model_parallel_region,
@@ -407,8 +408,7 @@ class RowParallelLinear(torch.nn.Module):
408 self.bias.zero_()
409 else:
410 self.register_parameter('bias', None)
410 -
411 -
411 + self.weight_t = self.weight.t()
412
413 def forward(self, input_):
414 """Forward of RowParallelLinear
@@ -425,11 +425,18 @@ class RowParallelLinear(torch.nn.Module):
425 input_parallel = input_
426 else:
427 input_parallel = scatter_to_tensor_model_parallel_region(input_)
428 - # Matrix multiply.
429 - output_parallel = F.linear(input_parallel, self.weight)
428 + if get_tensor_model_parallel_world_size() == 1:
429 + # Matrix multiply.
430 + output_ = F.linear(input_parallel, self.weight)
431 + else:
432 + # Matrix multiply.
433 + all_reduce_launcher = get_all_reduce_launcher()
434 + num_tokens = input_parallel.shape[0]
435 + output_buffer = all_reduce_launcher.buffer[:num_tokens]
436 + torch.matmul(input_parallel, self.weight_t, out=output_buffer)
437 + # All-reduce across all the partitions.
438 + output_ = all_reduce_launcher.launch(output_buffer)
439
431 - # All-reduce across all the partitions.
432 - output_ = reduce_from_tensor_model_parallel_region(output_parallel)
440 if not self.skip_bias_add:
441 output = output_ + self.bias if self.bias is not None else output_
442 output_bias = None
cacheflow/worker/controller.py
+2
@@ -27,6 +27,7 @@ class Controller:
27 dtype: str,
28 seed: int,
29 model_path: str,
30 + max_num_batched_tokens: int,
31 ) -> None:
32 self.stage_id = stage_id
33 self.stage_devices = stage_devices
@@ -57,6 +58,7 @@ class Controller:
58 tensor_parallel_size=tensor_parallel_size,
59 pipeline_parallel_size=pipeline_parallel_size,
60 model_path=model_path,
61 + max_num_batched_tokens=max_num_batched_tokens,
62 )
63 self.workers.append(worker)
64
cacheflow/worker/worker.py
+6 -1
@@ -9,7 +9,9 @@ from cacheflow.sequence import SequenceGroupInputs
9 from cacheflow.sequence import SequenceOutputs
10 from cacheflow.worker.cache_engine import CacheEngine
11 from cacheflow.parallel_utils.parallel_state import (
12 - initialize_model_parallel, get_tensor_model_parallel_world_size)
12 + initialize_model_parallel,
13 + initialize_all_reduce_launcher,
14 + get_tensor_model_parallel_world_size)
15 from cacheflow.utils import set_random_seed
16
17
@@ -27,6 +29,7 @@ class Worker:
29 rank: int,
30 world_size: int,
31 model_path: str,
32 + max_num_batched_tokens: int,
33 tensor_parallel_size: int = 1,
34 pipeline_parallel_size: int = 1,
35 ) -> None:
@@ -44,6 +47,8 @@ class Worker:
47 self.model = self.model.cuda()
48 tensor_model_parallel_world_size = (
49 get_tensor_model_parallel_world_size())
50 + initialize_all_reduce_launcher(
51 + max_num_batched_tokens, self.model.config.hidden_size, self.dtype)
52 self.num_layers = self.model.config.num_hidden_layers
53 assert self.model.config.num_attention_heads % tensor_model_parallel_world_size == 0
54 self.num_heads = self.model.config.num_attention_heads // tensor_model_parallel_world_size
simple_server.py
+1 -1
@@ -28,7 +28,7 @@ def main(args: argparse.Namespace):
28 dtype=args.dtype,
29 seed=args.seed,
30 swap_space=args.swap_space,
31 - max_batch_size=args.max_batch_size,
31 + max_num_batched_tokens=args.max_num_batched_tokens,
32 num_nodes=num_nodes,
33 num_devices_per_node=num_devices_per_node,
34 distributed_init_method=distributed_init_method,