A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Optimize tensor parallel execution speed (#17)
Zhuohan Li committed
Apr 1, 2023 at 00:51 UTC
c45f3c3ab60f4bf4eaab791a76028b8b07ffe9bd
3 files changed
+103
-287
benchmark/benchmark_latency.py
new
+99
@@ -0,0 +1,99 @@
1
+import argparse
2
+import time
3
+from typing import List
4
+
5
+from tqdm import tqdm
6
+import numpy as np
7
+import torch
8
+
9
+from cacheflow.master.simple_frontend import SimpleFrontend
10
+from cacheflow.master.server import (Server, add_server_arguments,
11
+ initialize_ray_cluster)
12
+from cacheflow.sampling_params import SamplingParams
13
+from cacheflow.utils import get_gpu_memory, get_cpu_memory
14
+
15
+
16
+def main(args: argparse.Namespace):
17
+ # TODO(zhuohan): Support pipeline parallelism.
18
+ assert args.pipeline_parallel_size == 1, (
19
+ 'Pipeline parallelism is not supported yet.')
20
+
21
+ (num_nodes, num_devices_per_node, distributed_init_method,
22
+ all_stage_devices) = (
23
+ initialize_ray_cluster(
24
+ address='local',
25
+ pipeline_parallel_size=args.pipeline_parallel_size,
26
+ tensor_parallel_size=args.tensor_parallel_size))
27
+
28
+ # Create a server.
29
+ server = Server(
30
+ model=args.model,
31
+ model_path=args.model_path,
32
+ pipeline_parallel_size=args.pipeline_parallel_size,
33
+ tensor_parallel_size=args.tensor_parallel_size,
34
+ block_size=args.block_size,
35
+ dtype=args.dtype,
36
+ seed=args.seed,
37
+ swap_space=args.swap_space,
38
+ max_batch_size=args.max_batch_size,
39
+ num_nodes=num_nodes,
40
+ num_devices_per_node=num_devices_per_node,
41
+ distributed_init_method=distributed_init_method,
42
+ all_stage_devices=all_stage_devices,
43
+ gpu_memory=get_gpu_memory(),
44
+ cpu_memory=get_cpu_memory(),
45
+ )
46
+
47
+ # Create a frontend.
48
+ frontend = SimpleFrontend(
49
+ model_name=args.model,
50
+ block_size=args.block_size,
51
+ )
52
+ sampling_params_dict = {
53
+ 'n': 1,
54
+ 'temperature': 0.0,
55
+ 'top_p': 1.0,
56
+ 'use_beam_search': False,
57
+ 'stop_token_ids': set(),
58
+ 'max_num_steps': args.output_len,
59
+ }
60
+ sampling_params = SamplingParams.from_dict(sampling_params_dict)
61
+ input_token_ids = [0] * args.input_len
62
+
63
+ def profile_step(profile=False):
64
+ if profile:
65
+ torch.cuda.cudart().cudaProfilerStart()
66
+ for _ in range(args.batch_size):
67
+ frontend._add_query(input_token_ids, sampling_params)
68
+ server.add_sequence_groups(frontend.get_inputs())
69
+ start_time = time.time()
70
+ while True:
71
+ server.step()
72
+ if not server.has_unfinished_requests():
73
+ break
74
+ end_time = time.time()
75
+ latency = end_time - start_time
76
+ if profile:
77
+ torch.cuda.cudart().cudaProfilerStop()
78
+ return latency
79
+
80
+ print("Warm up step")
81
+ profile_step()
82
+
83
+ # Benchmark.
84
+ latencies = []
85
+ for _ in tqdm(range(3), desc="Profile step"):
86
+ latencies.append(profile_step())
87
+ print(f'Avg latency: {np.mean(latencies)} seconds')
88
+
89
+
90
+if __name__ == '__main__':
91
+ parser = argparse.ArgumentParser(description='CacheFlow simple server.')
92
+ parser = add_server_arguments(parser)
93
+ parser.add_argument('--input-len', type=int, default=32)
94
+ parser.add_argument('--output-len', type=int, default=128)
95
+ parser.add_argument('--batch-size', type=int, default=8)
96
+ args = parser.parse_args()
97
+ args.max_batch_size = max(args.max_batch_size, args.batch_size * args.input_len)
98
+ print(args)
99
+ main(args)
cacheflow/parallel_utils/tensor_parallel/__init__.py
-3
@@ -6,8 +6,6 @@ from .layers import (
6
set_defaults_if_not_set_tensor_model_parallel_attributes,
7
copy_tensor_model_parallel_attributes,
8
param_is_not_tensor_parallel_duplicate,
9
- linear_with_grad_accumulation_and_async_allreduce
10
-
9
)
10
11
from .mappings import (
@@ -39,7 +37,6 @@ __all__ = [
37
"set_defaults_if_not_set_tensor_model_parallel_attributes",
38
"copy_tensor_model_parallel_attributes",
39
"param_is_not_tensor_parallel_duplicate",
42
- "linear_with_grad_accumulation_and_async_allreduce",
40
# mappings.py
41
"copy_to_tensor_model_parallel_region",
42
"gather_from_tensor_model_parallel_region",
cacheflow/parallel_utils/tensor_parallel/layers.py
+4
-284
@@ -3,10 +3,6 @@
3
# Parts of the code here are adapted from PyTorch
4
# repo: https://github.com/pytorch/pytorch
5
6
-import math
7
-import os
8
-from typing import Optional
9
-import warnings
6
7
import torch
8
import torch.nn.functional as F
@@ -16,31 +12,20 @@ from torch.nn.parameter import Parameter
12
from cacheflow.parallel_utils.parallel_state import (
13
get_tensor_model_parallel_rank,
14
get_tensor_model_parallel_world_size,
19
- get_tensor_model_parallel_group,
20
- get_global_memory_buffer,
15
)
16
from .mappings import (
17
copy_to_tensor_model_parallel_region,
18
gather_from_tensor_model_parallel_region,
25
- gather_from_sequence_parallel_region,
19
reduce_from_tensor_model_parallel_region,
20
scatter_to_tensor_model_parallel_region,
28
- reduce_scatter_to_sequence_parallel_region,
21
)
22
23
from .random import get_cuda_rng_tracker
24
from .utils import (
25
divide,
34
- split_tensor_along_last_dim,
26
VocabUtility,
27
)
28
38
-_grad_accum_fusion_available = True
39
-try:
40
- import fused_weight_gradient_mlp_cuda
41
-except ImportError:
42
- _grad_accum_fusion_available = False
43
-
29
_MODEL_PARALLEL_ATTRIBUTE_DEFAULTS = {'tensor_model_parallel': False,
30
'partition_dim': -1,
31
'partition_stride': 1}
@@ -216,202 +201,6 @@ class VocabParallelEmbedding(torch.nn.Module):
201
return output
202
203
219
-class LinearWithGradAccumulationAndAsyncCommunication(torch.autograd.Function):
220
- """See linear_with_grad_accumulation_and_async_allreduce"""
221
-
222
- @staticmethod
223
- def forward(ctx, input, weight, bias, gradient_accumulation_fusion,
224
- async_grad_allreduce, sequence_parallel):
225
- ctx.save_for_backward(input, weight)
226
- ctx.use_bias = bias is not None
227
- ctx.gradient_accumulation_fusion = gradient_accumulation_fusion
228
- ctx.async_grad_allreduce = async_grad_allreduce
229
- ctx.sequence_parallel = sequence_parallel
230
-
231
- if sequence_parallel:
232
- world_size = get_tensor_model_parallel_world_size()
233
- dim_size = list(input.size())
234
- dim_size[0] = dim_size[0] * world_size
235
-
236
- all_gather_buffer = \
237
- get_global_memory_buffer().get_tensor(dim_size, input.dtype, "mpu")
238
- torch.distributed._all_gather_base(
239
- all_gather_buffer,
240
- input,
241
- group=get_tensor_model_parallel_group())
242
- total_input = all_gather_buffer
243
- else:
244
- total_input = input
245
-
246
- output = torch.matmul(total_input, weight.t())
247
- if bias is not None:
248
- output = output + bias
249
- return output
250
-
251
- @staticmethod
252
- def backward(ctx, grad_output):
253
- input, weight = ctx.saved_tensors
254
- use_bias = ctx.use_bias
255
-
256
- if ctx.sequence_parallel:
257
- world_size = get_tensor_model_parallel_world_size()
258
- dim_size = list(input.size())
259
- dim_size[0] = dim_size[0] * world_size
260
-
261
- all_gather_buffer = \
262
- get_global_memory_buffer().get_tensor(dim_size, input.dtype, "mpu")
263
- handle = torch.distributed._all_gather_base(
264
- all_gather_buffer,
265
- input,
266
- group=get_tensor_model_parallel_group(), async_op=True)
267
-
268
- # Here we rely on CUDA_DEVICE_MAX_CONNECTIONS=1 to ensure that the
269
- # gather is scheduled before the input gradient computation
270
- total_input = all_gather_buffer
271
- else:
272
- total_input = input
273
- grad_input = grad_output.matmul(weight)
274
-
275
- if ctx.sequence_parallel:
276
- handle.wait()
277
-
278
- # Convert the tensor shapes to 2D for execution compatibility
279
- grad_output = grad_output.view(grad_output.shape[0] * grad_output.shape[1],
280
- grad_output.shape[2])
281
- total_input = total_input.view(total_input.shape[0] * total_input.shape[1],
282
- total_input.shape[2])
283
-
284
- if ctx.async_grad_allreduce:
285
- # Asynchronous all-reduce
286
- handle = torch.distributed.all_reduce(
287
- grad_input, group=get_tensor_model_parallel_group(), async_op=True)
288
- # Here we rely on CUDA_DEVICE_MAX_CONNECTIONS=1 to ensure that the
289
- # all-reduce is scheduled before the weight gradient computation
290
-
291
- if ctx.sequence_parallel:
292
- assert not ctx.async_grad_allreduce
293
- dim_size = list(input.size())
294
- sub_grad_input = torch.empty(dim_size, dtype=input.dtype,
295
- device=torch.cuda.current_device(),
296
- requires_grad=False)
297
- # reduce_scatter
298
- handle = torch.distributed._reduce_scatter_base(sub_grad_input, grad_input,
299
- group=get_tensor_model_parallel_group(),
300
- async_op=True)
301
- # Here we rely on CUDA_DEVICE_MAX_CONNECTIONS=1 to ensure that the
302
- # reduce scatter is scheduled before the weight gradient computation
303
-
304
-
305
- if ctx.gradient_accumulation_fusion:
306
- if weight.main_grad.dtype == torch.float32:
307
- fused_weight_gradient_mlp_cuda.wgrad_gemm_accum_fp32(total_input, grad_output, weight.main_grad)
308
- elif weight.main_grad.dtype == torch.float16:
309
- fused_weight_gradient_mlp_cuda.wgrad_gemm_accum_fp16(total_input, grad_output, weight.main_grad)
310
- else:
311
- raise RuntimeError("Unsupported gradient type for gradient accumulation fusion")
312
- grad_weight = None
313
- else:
314
- grad_weight = grad_output.t().matmul(total_input)
315
- grad_bias = grad_output.sum(dim=0) if use_bias else None
316
-
317
- if ctx.sequence_parallel:
318
- handle.wait()
319
- return sub_grad_input, grad_weight, grad_bias, None, None, None
320
-
321
- if ctx.async_grad_allreduce:
322
- handle.wait()
323
-
324
- return grad_input, grad_weight, grad_bias, None, None, None
325
-
326
-def linear_with_grad_accumulation_and_async_allreduce(
327
- input: torch.Tensor,
328
- weight: torch.Tensor,
329
- bias: Optional[torch.Tensor],
330
- gradient_accumulation_fusion: bool,
331
- async_grad_allreduce: bool,
332
- sequence_parallel_enabled: bool,
333
-) -> torch.Tensor:
334
- """Linear layer execution with asynchronous communication and
335
- gradient accumulation fusion in backprop.
336
-
337
- This has the option to accumulate the result of backprop
338
- calculation into an existing gradient buffer, preventing the need
339
- to do an additional addition kernel after the gradient
340
- calculation.
341
-
342
- Additionally, the tensor parallel all reduce of the input
343
- gradients can be done asynchronously with the calculation of
344
- the weight gradients.
345
-
346
- In the case of sequence parallelism, the reduce scatter of the
347
- input gradients is done asynchronously with the calcluation of the
348
- weight gradients.
349
-
350
- Use of this module requires that the environment variable
351
- CUDA_DEVICE_MAX_CONNECTIONS=1. There are a few collective
352
- operations, noted in the code, that should be scheduled before
353
- compute kernels to overlap the communication with the computation,
354
- which is necessary for a speedup but not for correctness so that
355
- ordering isn't imposed by the scheduler. Setting
356
- CUDA_DEVICE_MAX_CONNECTIONS=1 forces the kernels to be scheduled
357
- in the order they are called.
358
-
359
- Arguments:
360
-
361
- input (torch.Tensor required): input like torch.nn.functional.linear
362
-
363
- weight (torch.Tensor required): weight like torch.nn.functional.linear
364
-
365
- bias (torch.Tensor optional): bias like torch.nn.functional.linear
366
-
367
- gradient_accumulation_fusion (bool required): Perform the gradient
368
- accumulation fusion, requires the custom CUDA extension
369
- fused_weight_gradient_mlp_cuda module. To use
370
- gradient_accumulation_fusion you must install APEX with
371
- --cpp_ext and --cuda_ext. For example: "pip install
372
- --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext .\"
373
- " Note that the extension requires CUDA>=11. Otherwise, you
374
- must turn off gradient accumulation fusion."
375
-
376
- async_grad_allreduce (bool required): Do the allreduce of input
377
- gradients asyncronously with the computation of weight
378
- gradients. If sequence_parallel_enabled is True, this must be
379
- False, as no all reduce is performed.
380
-
381
- sequence_parallel_enabled (bool required): Indicates that sequence
382
- parallelism is used and thus in the forward pass the input is
383
- all gathered, and the backward pass the input gradients are
384
- reduce scattered.
385
- """
386
- args = [
387
- input,
388
- weight,
389
- bias,
390
- gradient_accumulation_fusion,
391
- async_grad_allreduce,
392
- sequence_parallel_enabled,
393
- ]
394
-
395
- if not linear_with_grad_accumulation_and_async_allreduce.warned:
396
- if os.environ.get('CUDA_DEVICE_MAX_CONNECTIONS') != "1":
397
- if sequence_parallel_enabled:
398
- warnings.warn(
399
- "When using sequence parallelism it is recommended to set the "
400
- "environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 for "
401
- "maximum speedup")
402
- linear_with_grad_accumulation_and_async_allreduce.warned = True
403
-
404
- if async_grad_allreduce:
405
- warnings.warn(
406
- "When using async grad allreduce it is recommended to set the "
407
- "environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 for "
408
- "maximum speedup")
409
- linear_with_grad_accumulation_and_async_allreduce.warned = True
410
-
411
- with torch.cuda.amp.autocast(enabled=False):
412
- return LinearWithGradAccumulationAndAsyncCommunication.apply(*args)
413
-linear_with_grad_accumulation_and_async_allreduce.warned = False
414
-
204
class ColumnParallelLinear(torch.nn.Module):
205
"""Linear layer with column parallelism.
206
@@ -436,11 +225,8 @@ class ColumnParallelLinear(torch.nn.Module):
225
skip_bias_add: This was added to enable performance optimations where bias
226
can be fused with other elementwise operations. we skip
227
adding bias but instead return it.
439
- async_tensor_model_parallel_allreduce:
228
params_dtype:
229
use_cpu_initialization:
442
- gradient_accumulation_fusion:
443
- sequence_parallel_enabled:
230
"""
231
232
def __init__(self, input_size, output_size, *,
@@ -448,12 +234,9 @@ class ColumnParallelLinear(torch.nn.Module):
234
init_method=init.xavier_normal_, stride=1,
235
keep_master_weight_for_test=False,
236
skip_bias_add=False,
451
- async_tensor_model_parallel_allreduce=True,
237
params_dtype=None,
238
use_cpu_initialization=False,
239
perform_initialization=True,
455
- gradient_accumulation_fusion=False,
456
- sequence_parallel_enabled: bool = False,
240
):
241
super(ColumnParallelLinear, self).__init__()
242
@@ -506,37 +289,6 @@ class ColumnParallelLinear(torch.nn.Module):
289
else:
290
self.register_parameter('bias', None)
291
509
- self.async_tensor_model_parallel_allreduce = (
510
- async_tensor_model_parallel_allreduce and
511
- world_size > 1)
512
- if sequence_parallel_enabled:
513
- if world_size <= 1:
514
- warnings.warn(
515
- f"`sequence_parallel_enabled` is set to `True`, but tensor model parallel size is {world_size}. "
516
- f"Disabling sequence parallel."
517
- )
518
- sequence_parallel_enabled = False
519
- self.sequence_parallel_enabled = sequence_parallel_enabled
520
-
521
- if gradient_accumulation_fusion:
522
- if not _grad_accum_fusion_available:
523
- raise RuntimeError(
524
- "ColumnParallelLinear was called with gradient_accumulation_fusion set "
525
- "to True but the custom CUDA extension fused_weight_gradient_mlp_cuda "
526
- "module is not found. To use gradient_accumulation_fusion you must "
527
- "install APEX with --cpp_ext and --cuda_ext. For example: "
528
- "pip install --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext .\" "
529
- "Note that the extension requires CUDA>=11. Otherwise, you must turn off "
530
- "gradient accumulation fusion."
531
- )
532
- self.gradient_accumulation_fusion = gradient_accumulation_fusion
533
-
534
- if self.async_tensor_model_parallel_allreduce and self.sequence_parallel_enabled:
535
- raise RuntimeError(
536
- "`async_tensor_model_parallel_allreduce` and `sequence_parallel_enabled` "
537
- "cannot be enabled at the same time."
538
- )
539
-
292
293
def forward(self, input_):
294
"""Forward of ColumnParallelLinear
@@ -550,23 +302,11 @@ class ColumnParallelLinear(torch.nn.Module):
302
"""
303
bias = self.bias if not self.skip_bias_add else None
304
553
- if self.async_tensor_model_parallel_allreduce or \
554
- self.sequence_parallel_enabled:
555
- input_parallel = input_
556
- else:
557
- input_parallel = copy_to_tensor_model_parallel_region(input_)
305
+ input_parallel = copy_to_tensor_model_parallel_region(input_)
306
# Matrix multiply.
559
- output_parallel = linear_with_grad_accumulation_and_async_allreduce(
560
- input=input_parallel,
561
- weight=self.weight,
562
- bias=bias,
563
- gradient_accumulation_fusion=self.gradient_accumulation_fusion,
564
- async_grad_allreduce=self.async_tensor_model_parallel_allreduce,
565
- sequence_parallel_enabled=self.sequence_parallel_enabled,
566
- )
307
+ output_parallel = F.linear(input_parallel, self.weight, bias)
308
if self.gather_output:
309
# All-gather across the partitions.
569
- assert not self.sequence_parallel_enabled
310
output = gather_from_tensor_model_parallel_region(output_parallel)
311
else:
312
output = output_parallel
@@ -607,8 +347,6 @@ class RowParallelLinear(torch.nn.Module):
347
params_dtype:
348
use_cpu_initialization:
349
perform_initialization:
610
- gradient_accumulation_fusion:
611
- sequence_parallel_enabled:
350
"""
351
352
def __init__(self, input_size, output_size, *,
@@ -619,8 +357,6 @@ class RowParallelLinear(torch.nn.Module):
357
params_dtype=None,
358
use_cpu_initialization=False,
359
perform_initialization=True,
622
- gradient_accumulation_fusion=False,
623
- sequence_parallel_enabled: bool = False,
360
):
361
super(RowParallelLinear, self).__init__()
362
@@ -635,10 +371,6 @@ class RowParallelLinear(torch.nn.Module):
371
world_size = get_tensor_model_parallel_world_size()
372
self.input_size_per_partition = divide(input_size, world_size)
373
self.skip_bias_add = skip_bias_add
638
- self.gradient_accumulation_fusion = gradient_accumulation_fusion
639
- self.sequence_parallel_enabled = sequence_parallel_enabled
640
- if self.sequence_parallel_enabled and not self.input_is_parallel:
641
- raise RuntimeError("To enable `sequence_parallel_enabled`, `input_is_parallel` must be `True`")
374
375
# Parameters.
376
# Note: torch.nn.functional.linear performs XA^T + b and as a result
@@ -669,7 +401,6 @@ class RowParallelLinear(torch.nn.Module):
401
self.bias = Parameter(torch.empty(
402
self.output_size, device=torch.cuda.current_device(),
403
dtype=params_dtype))
672
- setattr(self.bias, 'sequence_parallel', sequence_parallel_enabled)
404
405
# Always initialize bias to zero.
406
with torch.no_grad():
@@ -693,23 +424,12 @@ class RowParallelLinear(torch.nn.Module):
424
if self.input_is_parallel:
425
input_parallel = input_
426
else:
696
- assert not self.sequence_parallel_enabled
427
input_parallel = scatter_to_tensor_model_parallel_region(input_)
428
# Matrix multiply.
699
- output_parallel = linear_with_grad_accumulation_and_async_allreduce(
700
- input=input_parallel,
701
- weight=self.weight,
702
- bias=None,
703
- gradient_accumulation_fusion=self.gradient_accumulation_fusion,
704
- async_grad_allreduce=False,
705
- sequence_parallel_enabled=False,
706
- )
429
+ output_parallel = F.linear(input_parallel, self.weight)
430
431
# All-reduce across all the partitions.
709
- if self.sequence_parallel_enabled:
710
- output_ = reduce_scatter_to_sequence_parallel_region(output_parallel)
711
- else:
712
- output_ = reduce_from_tensor_model_parallel_region(output_parallel)
432
+ output_ = reduce_from_tensor_model_parallel_region(output_parallel)
433
if not self.skip_bias_add:
434
output = output_ + self.bias if self.bias is not None else output_
435
output_bias = None