@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c45f3c3ab6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Optimize tensor parallel execution speed (#17)

Zhuohan Li committed Apr 1, 2023 at 00:51 UTC c45f3c3ab60f4bf4eaab791a76028b8b07ffe9bd
3 files changed +103 -287
benchmark/benchmark_latency.py new
+99
@@ -0,0 +1,99 @@
1 +import argparse
2 +import time
3 +from typing import List
4 +
5 +from tqdm import tqdm
6 +import numpy as np
7 +import torch
8 +
9 +from cacheflow.master.simple_frontend import SimpleFrontend
10 +from cacheflow.master.server import (Server, add_server_arguments,
11 + initialize_ray_cluster)
12 +from cacheflow.sampling_params import SamplingParams
13 +from cacheflow.utils import get_gpu_memory, get_cpu_memory
14 +
15 +
16 +def main(args: argparse.Namespace):
17 + # TODO(zhuohan): Support pipeline parallelism.
18 + assert args.pipeline_parallel_size == 1, (
19 + 'Pipeline parallelism is not supported yet.')
20 +
21 + (num_nodes, num_devices_per_node, distributed_init_method,
22 + all_stage_devices) = (
23 + initialize_ray_cluster(
24 + address='local',
25 + pipeline_parallel_size=args.pipeline_parallel_size,
26 + tensor_parallel_size=args.tensor_parallel_size))
27 +
28 + # Create a server.
29 + server = Server(
30 + model=args.model,
31 + model_path=args.model_path,
32 + pipeline_parallel_size=args.pipeline_parallel_size,
33 + tensor_parallel_size=args.tensor_parallel_size,
34 + block_size=args.block_size,
35 + dtype=args.dtype,
36 + seed=args.seed,
37 + swap_space=args.swap_space,
38 + max_batch_size=args.max_batch_size,
39 + num_nodes=num_nodes,
40 + num_devices_per_node=num_devices_per_node,
41 + distributed_init_method=distributed_init_method,
42 + all_stage_devices=all_stage_devices,
43 + gpu_memory=get_gpu_memory(),
44 + cpu_memory=get_cpu_memory(),
45 + )
46 +
47 + # Create a frontend.
48 + frontend = SimpleFrontend(
49 + model_name=args.model,
50 + block_size=args.block_size,
51 + )
52 + sampling_params_dict = {
53 + 'n': 1,
54 + 'temperature': 0.0,
55 + 'top_p': 1.0,
56 + 'use_beam_search': False,
57 + 'stop_token_ids': set(),
58 + 'max_num_steps': args.output_len,
59 + }
60 + sampling_params = SamplingParams.from_dict(sampling_params_dict)
61 + input_token_ids = [0] * args.input_len
62 +
63 + def profile_step(profile=False):
64 + if profile:
65 + torch.cuda.cudart().cudaProfilerStart()
66 + for _ in range(args.batch_size):
67 + frontend._add_query(input_token_ids, sampling_params)
68 + server.add_sequence_groups(frontend.get_inputs())
69 + start_time = time.time()
70 + while True:
71 + server.step()
72 + if not server.has_unfinished_requests():
73 + break
74 + end_time = time.time()
75 + latency = end_time - start_time
76 + if profile:
77 + torch.cuda.cudart().cudaProfilerStop()
78 + return latency
79 +
80 + print("Warm up step")
81 + profile_step()
82 +
83 + # Benchmark.
84 + latencies = []
85 + for _ in tqdm(range(3), desc="Profile step"):
86 + latencies.append(profile_step())
87 + print(f'Avg latency: {np.mean(latencies)} seconds')
88 +
89 +
90 +if __name__ == '__main__':
91 + parser = argparse.ArgumentParser(description='CacheFlow simple server.')
92 + parser = add_server_arguments(parser)
93 + parser.add_argument('--input-len', type=int, default=32)
94 + parser.add_argument('--output-len', type=int, default=128)
95 + parser.add_argument('--batch-size', type=int, default=8)
96 + args = parser.parse_args()
97 + args.max_batch_size = max(args.max_batch_size, args.batch_size * args.input_len)
98 + print(args)
99 + main(args)
cacheflow/parallel_utils/tensor_parallel/__init__.py
-3
@@ -6,8 +6,6 @@ from .layers import (
6 set_defaults_if_not_set_tensor_model_parallel_attributes,
7 copy_tensor_model_parallel_attributes,
8 param_is_not_tensor_parallel_duplicate,
9 - linear_with_grad_accumulation_and_async_allreduce
10 -
9 )
10
11 from .mappings import (
@@ -39,7 +37,6 @@ __all__ = [
37 "set_defaults_if_not_set_tensor_model_parallel_attributes",
38 "copy_tensor_model_parallel_attributes",
39 "param_is_not_tensor_parallel_duplicate",
42 - "linear_with_grad_accumulation_and_async_allreduce",
40 # mappings.py
41 "copy_to_tensor_model_parallel_region",
42 "gather_from_tensor_model_parallel_region",
cacheflow/parallel_utils/tensor_parallel/layers.py
+4 -284
@@ -3,10 +3,6 @@
3 # Parts of the code here are adapted from PyTorch
4 # repo: https://github.com/pytorch/pytorch
5
6 -import math
7 -import os
8 -from typing import Optional
9 -import warnings
6
7 import torch
8 import torch.nn.functional as F
@@ -16,31 +12,20 @@ from torch.nn.parameter import Parameter
12 from cacheflow.parallel_utils.parallel_state import (
13 get_tensor_model_parallel_rank,
14 get_tensor_model_parallel_world_size,
19 - get_tensor_model_parallel_group,
20 - get_global_memory_buffer,
15 )
16 from .mappings import (
17 copy_to_tensor_model_parallel_region,
18 gather_from_tensor_model_parallel_region,
25 - gather_from_sequence_parallel_region,
19 reduce_from_tensor_model_parallel_region,
20 scatter_to_tensor_model_parallel_region,
28 - reduce_scatter_to_sequence_parallel_region,
21 )
22
23 from .random import get_cuda_rng_tracker
24 from .utils import (
25 divide,
34 - split_tensor_along_last_dim,
26 VocabUtility,
27 )
28
38 -_grad_accum_fusion_available = True
39 -try:
40 - import fused_weight_gradient_mlp_cuda
41 -except ImportError:
42 - _grad_accum_fusion_available = False
43 -
29 _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS = {'tensor_model_parallel': False,
30 'partition_dim': -1,
31 'partition_stride': 1}
@@ -216,202 +201,6 @@ class VocabParallelEmbedding(torch.nn.Module):
201 return output
202
203
219 -class LinearWithGradAccumulationAndAsyncCommunication(torch.autograd.Function):
220 - """See linear_with_grad_accumulation_and_async_allreduce"""
221 -
222 - @staticmethod
223 - def forward(ctx, input, weight, bias, gradient_accumulation_fusion,
224 - async_grad_allreduce, sequence_parallel):
225 - ctx.save_for_backward(input, weight)
226 - ctx.use_bias = bias is not None
227 - ctx.gradient_accumulation_fusion = gradient_accumulation_fusion
228 - ctx.async_grad_allreduce = async_grad_allreduce
229 - ctx.sequence_parallel = sequence_parallel
230 -
231 - if sequence_parallel:
232 - world_size = get_tensor_model_parallel_world_size()
233 - dim_size = list(input.size())
234 - dim_size[0] = dim_size[0] * world_size
235 -
236 - all_gather_buffer = \
237 - get_global_memory_buffer().get_tensor(dim_size, input.dtype, "mpu")
238 - torch.distributed._all_gather_base(
239 - all_gather_buffer,
240 - input,
241 - group=get_tensor_model_parallel_group())
242 - total_input = all_gather_buffer
243 - else:
244 - total_input = input
245 -
246 - output = torch.matmul(total_input, weight.t())
247 - if bias is not None:
248 - output = output + bias
249 - return output
250 -
251 - @staticmethod
252 - def backward(ctx, grad_output):
253 - input, weight = ctx.saved_tensors
254 - use_bias = ctx.use_bias
255 -
256 - if ctx.sequence_parallel:
257 - world_size = get_tensor_model_parallel_world_size()
258 - dim_size = list(input.size())
259 - dim_size[0] = dim_size[0] * world_size
260 -
261 - all_gather_buffer = \
262 - get_global_memory_buffer().get_tensor(dim_size, input.dtype, "mpu")
263 - handle = torch.distributed._all_gather_base(
264 - all_gather_buffer,
265 - input,
266 - group=get_tensor_model_parallel_group(), async_op=True)
267 -
268 - # Here we rely on CUDA_DEVICE_MAX_CONNECTIONS=1 to ensure that the
269 - # gather is scheduled before the input gradient computation
270 - total_input = all_gather_buffer
271 - else:
272 - total_input = input
273 - grad_input = grad_output.matmul(weight)
274 -
275 - if ctx.sequence_parallel:
276 - handle.wait()
277 -
278 - # Convert the tensor shapes to 2D for execution compatibility
279 - grad_output = grad_output.view(grad_output.shape[0] * grad_output.shape[1],
280 - grad_output.shape[2])
281 - total_input = total_input.view(total_input.shape[0] * total_input.shape[1],
282 - total_input.shape[2])
283 -
284 - if ctx.async_grad_allreduce:
285 - # Asynchronous all-reduce
286 - handle = torch.distributed.all_reduce(
287 - grad_input, group=get_tensor_model_parallel_group(), async_op=True)
288 - # Here we rely on CUDA_DEVICE_MAX_CONNECTIONS=1 to ensure that the
289 - # all-reduce is scheduled before the weight gradient computation
290 -
291 - if ctx.sequence_parallel:
292 - assert not ctx.async_grad_allreduce
293 - dim_size = list(input.size())
294 - sub_grad_input = torch.empty(dim_size, dtype=input.dtype,
295 - device=torch.cuda.current_device(),
296 - requires_grad=False)
297 - # reduce_scatter
298 - handle = torch.distributed._reduce_scatter_base(sub_grad_input, grad_input,
299 - group=get_tensor_model_parallel_group(),
300 - async_op=True)
301 - # Here we rely on CUDA_DEVICE_MAX_CONNECTIONS=1 to ensure that the
302 - # reduce scatter is scheduled before the weight gradient computation
303 -
304 -
305 - if ctx.gradient_accumulation_fusion:
306 - if weight.main_grad.dtype == torch.float32:
307 - fused_weight_gradient_mlp_cuda.wgrad_gemm_accum_fp32(total_input, grad_output, weight.main_grad)
308 - elif weight.main_grad.dtype == torch.float16:
309 - fused_weight_gradient_mlp_cuda.wgrad_gemm_accum_fp16(total_input, grad_output, weight.main_grad)
310 - else:
311 - raise RuntimeError("Unsupported gradient type for gradient accumulation fusion")
312 - grad_weight = None
313 - else:
314 - grad_weight = grad_output.t().matmul(total_input)
315 - grad_bias = grad_output.sum(dim=0) if use_bias else None
316 -
317 - if ctx.sequence_parallel:
318 - handle.wait()
319 - return sub_grad_input, grad_weight, grad_bias, None, None, None
320 -
321 - if ctx.async_grad_allreduce:
322 - handle.wait()
323 -
324 - return grad_input, grad_weight, grad_bias, None, None, None
325 -
326 -def linear_with_grad_accumulation_and_async_allreduce(
327 - input: torch.Tensor,
328 - weight: torch.Tensor,
329 - bias: Optional[torch.Tensor],
330 - gradient_accumulation_fusion: bool,
331 - async_grad_allreduce: bool,
332 - sequence_parallel_enabled: bool,
333 -) -> torch.Tensor:
334 - """Linear layer execution with asynchronous communication and
335 - gradient accumulation fusion in backprop.
336 -
337 - This has the option to accumulate the result of backprop
338 - calculation into an existing gradient buffer, preventing the need
339 - to do an additional addition kernel after the gradient
340 - calculation.
341 -
342 - Additionally, the tensor parallel all reduce of the input
343 - gradients can be done asynchronously with the calculation of
344 - the weight gradients.
345 -
346 - In the case of sequence parallelism, the reduce scatter of the
347 - input gradients is done asynchronously with the calcluation of the
348 - weight gradients.
349 -
350 - Use of this module requires that the environment variable
351 - CUDA_DEVICE_MAX_CONNECTIONS=1. There are a few collective
352 - operations, noted in the code, that should be scheduled before
353 - compute kernels to overlap the communication with the computation,
354 - which is necessary for a speedup but not for correctness so that
355 - ordering isn't imposed by the scheduler. Setting
356 - CUDA_DEVICE_MAX_CONNECTIONS=1 forces the kernels to be scheduled
357 - in the order they are called.
358 -
359 - Arguments:
360 -
361 - input (torch.Tensor required): input like torch.nn.functional.linear
362 -
363 - weight (torch.Tensor required): weight like torch.nn.functional.linear
364 -
365 - bias (torch.Tensor optional): bias like torch.nn.functional.linear
366 -
367 - gradient_accumulation_fusion (bool required): Perform the gradient
368 - accumulation fusion, requires the custom CUDA extension
369 - fused_weight_gradient_mlp_cuda module. To use
370 - gradient_accumulation_fusion you must install APEX with
371 - --cpp_ext and --cuda_ext. For example: "pip install
372 - --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext .\"
373 - " Note that the extension requires CUDA>=11. Otherwise, you
374 - must turn off gradient accumulation fusion."
375 -
376 - async_grad_allreduce (bool required): Do the allreduce of input
377 - gradients asyncronously with the computation of weight
378 - gradients. If sequence_parallel_enabled is True, this must be
379 - False, as no all reduce is performed.
380 -
381 - sequence_parallel_enabled (bool required): Indicates that sequence
382 - parallelism is used and thus in the forward pass the input is
383 - all gathered, and the backward pass the input gradients are
384 - reduce scattered.
385 - """
386 - args = [
387 - input,
388 - weight,
389 - bias,
390 - gradient_accumulation_fusion,
391 - async_grad_allreduce,
392 - sequence_parallel_enabled,
393 - ]
394 -
395 - if not linear_with_grad_accumulation_and_async_allreduce.warned:
396 - if os.environ.get('CUDA_DEVICE_MAX_CONNECTIONS') != "1":
397 - if sequence_parallel_enabled:
398 - warnings.warn(
399 - "When using sequence parallelism it is recommended to set the "
400 - "environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 for "
401 - "maximum speedup")
402 - linear_with_grad_accumulation_and_async_allreduce.warned = True
403 -
404 - if async_grad_allreduce:
405 - warnings.warn(
406 - "When using async grad allreduce it is recommended to set the "
407 - "environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 for "
408 - "maximum speedup")
409 - linear_with_grad_accumulation_and_async_allreduce.warned = True
410 -
411 - with torch.cuda.amp.autocast(enabled=False):
412 - return LinearWithGradAccumulationAndAsyncCommunication.apply(*args)
413 -linear_with_grad_accumulation_and_async_allreduce.warned = False
414 -
204 class ColumnParallelLinear(torch.nn.Module):
205 """Linear layer with column parallelism.
206
@@ -436,11 +225,8 @@ class ColumnParallelLinear(torch.nn.Module):
225 skip_bias_add: This was added to enable performance optimations where bias
226 can be fused with other elementwise operations. we skip
227 adding bias but instead return it.
439 - async_tensor_model_parallel_allreduce:
228 params_dtype:
229 use_cpu_initialization:
442 - gradient_accumulation_fusion:
443 - sequence_parallel_enabled:
230 """
231
232 def __init__(self, input_size, output_size, *,
@@ -448,12 +234,9 @@ class ColumnParallelLinear(torch.nn.Module):
234 init_method=init.xavier_normal_, stride=1,
235 keep_master_weight_for_test=False,
236 skip_bias_add=False,
451 - async_tensor_model_parallel_allreduce=True,
237 params_dtype=None,
238 use_cpu_initialization=False,
239 perform_initialization=True,
455 - gradient_accumulation_fusion=False,
456 - sequence_parallel_enabled: bool = False,
240 ):
241 super(ColumnParallelLinear, self).__init__()
242
@@ -506,37 +289,6 @@ class ColumnParallelLinear(torch.nn.Module):
289 else:
290 self.register_parameter('bias', None)
291
509 - self.async_tensor_model_parallel_allreduce = (
510 - async_tensor_model_parallel_allreduce and
511 - world_size > 1)
512 - if sequence_parallel_enabled:
513 - if world_size <= 1:
514 - warnings.warn(
515 - f"`sequence_parallel_enabled` is set to `True`, but tensor model parallel size is {world_size}. "
516 - f"Disabling sequence parallel."
517 - )
518 - sequence_parallel_enabled = False
519 - self.sequence_parallel_enabled = sequence_parallel_enabled
520 -
521 - if gradient_accumulation_fusion:
522 - if not _grad_accum_fusion_available:
523 - raise RuntimeError(
524 - "ColumnParallelLinear was called with gradient_accumulation_fusion set "
525 - "to True but the custom CUDA extension fused_weight_gradient_mlp_cuda "
526 - "module is not found. To use gradient_accumulation_fusion you must "
527 - "install APEX with --cpp_ext and --cuda_ext. For example: "
528 - "pip install --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext .\" "
529 - "Note that the extension requires CUDA>=11. Otherwise, you must turn off "
530 - "gradient accumulation fusion."
531 - )
532 - self.gradient_accumulation_fusion = gradient_accumulation_fusion
533 -
534 - if self.async_tensor_model_parallel_allreduce and self.sequence_parallel_enabled:
535 - raise RuntimeError(
536 - "`async_tensor_model_parallel_allreduce` and `sequence_parallel_enabled` "
537 - "cannot be enabled at the same time."
538 - )
539 -
292
293 def forward(self, input_):
294 """Forward of ColumnParallelLinear
@@ -550,23 +302,11 @@ class ColumnParallelLinear(torch.nn.Module):
302 """
303 bias = self.bias if not self.skip_bias_add else None
304
553 - if self.async_tensor_model_parallel_allreduce or \
554 - self.sequence_parallel_enabled:
555 - input_parallel = input_
556 - else:
557 - input_parallel = copy_to_tensor_model_parallel_region(input_)
305 + input_parallel = copy_to_tensor_model_parallel_region(input_)
306 # Matrix multiply.
559 - output_parallel = linear_with_grad_accumulation_and_async_allreduce(
560 - input=input_parallel,
561 - weight=self.weight,
562 - bias=bias,
563 - gradient_accumulation_fusion=self.gradient_accumulation_fusion,
564 - async_grad_allreduce=self.async_tensor_model_parallel_allreduce,
565 - sequence_parallel_enabled=self.sequence_parallel_enabled,
566 - )
307 + output_parallel = F.linear(input_parallel, self.weight, bias)
308 if self.gather_output:
309 # All-gather across the partitions.
569 - assert not self.sequence_parallel_enabled
310 output = gather_from_tensor_model_parallel_region(output_parallel)
311 else:
312 output = output_parallel
@@ -607,8 +347,6 @@ class RowParallelLinear(torch.nn.Module):
347 params_dtype:
348 use_cpu_initialization:
349 perform_initialization:
610 - gradient_accumulation_fusion:
611 - sequence_parallel_enabled:
350 """
351
352 def __init__(self, input_size, output_size, *,
@@ -619,8 +357,6 @@ class RowParallelLinear(torch.nn.Module):
357 params_dtype=None,
358 use_cpu_initialization=False,
359 perform_initialization=True,
622 - gradient_accumulation_fusion=False,
623 - sequence_parallel_enabled: bool = False,
360 ):
361 super(RowParallelLinear, self).__init__()
362
@@ -635,10 +371,6 @@ class RowParallelLinear(torch.nn.Module):
371 world_size = get_tensor_model_parallel_world_size()
372 self.input_size_per_partition = divide(input_size, world_size)
373 self.skip_bias_add = skip_bias_add
638 - self.gradient_accumulation_fusion = gradient_accumulation_fusion
639 - self.sequence_parallel_enabled = sequence_parallel_enabled
640 - if self.sequence_parallel_enabled and not self.input_is_parallel:
641 - raise RuntimeError("To enable `sequence_parallel_enabled`, `input_is_parallel` must be `True`")
374
375 # Parameters.
376 # Note: torch.nn.functional.linear performs XA^T + b and as a result
@@ -669,7 +401,6 @@ class RowParallelLinear(torch.nn.Module):
401 self.bias = Parameter(torch.empty(
402 self.output_size, device=torch.cuda.current_device(),
403 dtype=params_dtype))
672 - setattr(self.bias, 'sequence_parallel', sequence_parallel_enabled)
404
405 # Always initialize bias to zero.
406 with torch.no_grad():
@@ -693,23 +424,12 @@ class RowParallelLinear(torch.nn.Module):
424 if self.input_is_parallel:
425 input_parallel = input_
426 else:
696 - assert not self.sequence_parallel_enabled
427 input_parallel = scatter_to_tensor_model_parallel_region(input_)
428 # Matrix multiply.
699 - output_parallel = linear_with_grad_accumulation_and_async_allreduce(
700 - input=input_parallel,
701 - weight=self.weight,
702 - bias=None,
703 - gradient_accumulation_fusion=self.gradient_accumulation_fusion,
704 - async_grad_allreduce=False,
705 - sequence_parallel_enabled=False,
706 - )
429 + output_parallel = F.linear(input_parallel, self.weight)
430
431 # All-reduce across all the partitions.
709 - if self.sequence_parallel_enabled:
710 - output_ = reduce_scatter_to_sequence_parallel_region(output_parallel)
711 - else:
712 - output_ = reduce_from_tensor_model_parallel_region(output_parallel)
432 + output_ = reduce_from_tensor_model_parallel_region(output_parallel)
433 if not self.skip_bias_add:
434 output = output_ + self.bias if self.bias is not None else output_
435 output_bias = None