A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Remove unused parts in Megatron-LM code and add copyright notice (#110)
Zhuohan Li committed
May 20, 2023 at 09:11 UTC
7297fa6f7c4ce6413ee005025b312c4c9d4f5f0b
8 files changed
+23
-297
cacheflow/model_executor/parallel_utils/__init__.py
-2
@@ -1,6 +1,5 @@
1
import cacheflow.model_executor.parallel_utils.parallel_state
2
import cacheflow.model_executor.parallel_utils.tensor_parallel
3
-import cacheflow.model_executor.parallel_utils.utils
3
4
# Alias parallel_state as mpu, its legacy name
5
mpu = parallel_state
@@ -8,5 +7,4 @@ mpu = parallel_state
7
__all__ = [
8
"parallel_state",
9
"tensor_parallel",
11
- "utils",
10
]
cacheflow/model_executor/parallel_utils/parallel_state.py
+2
-25
@@ -1,3 +1,5 @@
1
+# Copyright 2023 The CacheFlow team.
2
+# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
3
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
5
"""Model and data parallel groups."""
@@ -5,8 +7,6 @@
7
import torch
8
from typing import Optional
9
8
-from .utils import GlobalMemoryBuffer
9
-
10
# Intra-layer model parallel group that the current rank belongs to.
11
_TENSOR_MODEL_PARALLEL_GROUP = None
12
# Inter-layer model parallel group that the current rank belongs to.
@@ -44,9 +44,6 @@ _PIPELINE_GLOBAL_RANKS = None
44
# rank when broadcasting weights from src to all other data parallel ranks
45
_DATA_PARALLEL_GLOBAL_RANKS = None
46
47
-# Memory buffers to avoid dynamic memory allocation
48
-_GLOBAL_MEMORY_BUFFER = None
49
-
47
_ALL_REDUCE_LAUNCHER: Optional['GraphAllReduce'] = None
48
49
def initialize_model_parallel(
@@ -199,13 +196,6 @@ def initialize_model_parallel(
196
if rank in ranks:
197
_POSITION_EMBEDDING_GLOBAL_RANKS = position_embedding_ranks
198
202
- # Initialize global memory buffer
203
- # This isn't really "parallel state" but there isn't another good place to
204
- # put this. If we end up with a more generic initialization of megatron-core
205
- # we could stick it there
206
- _set_global_memory_buffer()
207
-
208
-
199
def initialize_all_reduce_launcher(
200
max_num_tokens: int,
201
hidden_size: int,
@@ -495,17 +485,6 @@ def get_data_parallel_rank():
485
"""Return my rank for the data parallel group."""
486
return torch.distributed.get_rank(group=get_data_parallel_group())
487
498
-def _set_global_memory_buffer():
499
- """Initialize global buffer"""
500
- global _GLOBAL_MEMORY_BUFFER
501
- assert _GLOBAL_MEMORY_BUFFER is None, 'global memory buffer is already initialized'
502
- _GLOBAL_MEMORY_BUFFER = GlobalMemoryBuffer()
503
-
504
-def get_global_memory_buffer():
505
- """Return the global GlobalMemoryBuffer object"""
506
- assert _GLOBAL_MEMORY_BUFFER is not None, 'global memory buffer is not initialized'
507
- return _GLOBAL_MEMORY_BUFFER
508
-
488
def get_all_reduce_launcher() -> 'GraphAllReduce':
489
assert _ALL_REDUCE_LAUNCHER is not None, 'all reduce launcher is not initialized'
490
return _ALL_REDUCE_LAUNCHER
@@ -536,8 +515,6 @@ def destroy_model_parallel():
515
_MPU_TENSOR_MODEL_PARALLEL_RANK = None
516
global _MPU_PIPELINE_MODEL_PARALLEL_RANK
517
_MPU_PIPELINE_MODEL_PARALLEL_RANK = None
539
- global _GLOBAL_MEMORY_BUFFER
540
- _GLOBAL_MEMORY_BUFFER = None
518
519
520
class GraphAllReduce:
cacheflow/model_executor/parallel_utils/tensor_parallel/__init__.py
-6
@@ -17,15 +17,12 @@ from .mappings import (
17
)
18
19
from .random import (
20
- checkpoint,
20
get_cuda_rng_tracker,
21
model_parallel_cuda_manual_seed,
22
)
23
24
from .utils import (
25
split_tensor_along_last_dim,
27
- split_tensor_into_1d_equal_chunks,
28
- gather_split_1d_tensor,
26
)
27
28
__all__ = [
@@ -45,11 +42,8 @@ __all__ = [
42
"scatter_to_tensor_model_parallel_region",
43
"scatter_to_sequence_parallel_region",
44
# random.py
48
- "checkpoint",
45
"get_cuda_rng_tracker",
46
"model_parallel_cuda_manual_seed",
47
# utils.py
48
"split_tensor_along_last_dim",
53
- "split_tensor_into_1d_equal_chunks",
54
- "gather_split_1d_tensor",
49
]
cacheflow/model_executor/parallel_utils/tensor_parallel/layers.py
+2
@@ -1,3 +1,5 @@
1
+# Copyright 2023 The CacheFlow team.
2
+# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/layers.py
3
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
5
# Parts of the code here are adapted from PyTorch
cacheflow/model_executor/parallel_utils/tensor_parallel/mappings.py
+2
@@ -1,3 +1,5 @@
1
+# Copyright 2023 The CacheFlow team.
2
+# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/mappings.py
3
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
5
import torch
cacheflow/model_executor/parallel_utils/tensor_parallel/random.py
+2
-91
@@ -1,3 +1,5 @@
1
+# Copyright 2023 The CacheFlow team.
2
+# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/random.py
3
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
5
# Parts of the code here are adapted from PyTorch
@@ -8,22 +10,11 @@ import contextlib
10
import torch
11
from torch import _C
12
from torch.cuda import _lazy_call, device as device_ctx_manager
11
-from torch.utils.checkpoint import detach_variable
13
14
from cacheflow.model_executor.parallel_utils.parallel_state import (
14
- get_data_parallel_rank,
15
- get_tensor_model_parallel_group,
15
get_tensor_model_parallel_rank,
17
- get_tensor_model_parallel_world_size,
16
)
17
20
-from .utils import (
21
- split_tensor_into_1d_equal_chunks,
22
- gather_split_1d_tensor,
23
-)
24
-
25
-from cacheflow.model_executor.parallel_utils.utils import safely_set_viewless_tensor_data
26
-
18
# Default name for the model parallel rng tracker.
19
_MODEL_PARALLEL_RNG_TRACKER_NAME = 'model-parallel-rng'
20
@@ -171,83 +162,3 @@ def model_parallel_cuda_manual_seed(seed):
162
# and model parallel state.
163
_CUDA_RNG_STATE_TRACKER.add(_MODEL_PARALLEL_RNG_TRACKER_NAME,
164
tensor_model_parallel_seed)
174
-
175
-
176
-class CheckpointFunction(torch.autograd.Function):
177
- """This function is adapted from torch.utils.checkpoint with
178
- two main changes:
179
- 1) torch.cuda.set_rng_state is replaced with `_set_cuda_rng_state`
180
- 2) the states in the model parallel tracker are also properly
181
- tracked/set/reset.
182
- """
183
- @staticmethod
184
- def forward(ctx, run_function, distribute_saved_activations, *args):
185
- ctx.run_function = run_function
186
- ctx.distribute_saved_activations \
187
- = distribute_saved_activations
188
-
189
- # Copy the rng states.
190
- ctx.fwd_cpu_rng_state = torch.get_rng_state()
191
- ctx.fwd_cuda_rng_state = torch.cuda.get_rng_state()
192
- ctx.fwd_cuda_rng_state_tracker = get_cuda_rng_tracker().get_states()
193
-
194
- with torch.no_grad():
195
- outputs = run_function(*args)
196
-
197
- # Divide hidden states across model parallel group and only keep
198
- # the chunk corresponding to the current rank.
199
- if distribute_saved_activations:
200
- ctx.input_0_shape = args[0].data.shape
201
- safely_set_viewless_tensor_data(
202
- args[0],
203
- split_tensor_into_1d_equal_chunks(args[0].data, new_buffer=True))
204
-
205
- # Store everything.
206
- ctx.save_for_backward(*args)
207
-
208
- return outputs
209
-
210
- @staticmethod
211
- def backward(ctx, *args):
212
- if not torch.autograd._is_checkpoint_valid():
213
- raise RuntimeError("Checkpointing is not compatible with .grad(), "
214
- "please use .backward() if possible")
215
- inputs = ctx.saved_tensors
216
- if ctx.distribute_saved_activations:
217
- safely_set_viewless_tensor_data(
218
- inputs[0],
219
- gather_split_1d_tensor(inputs[0].data).view(ctx.input_0_shape))
220
-
221
- # Store the current states.
222
- bwd_cpu_rng_state = torch.get_rng_state()
223
- bwd_cuda_rng_state = torch.cuda.get_rng_state()
224
- bwd_cuda_rng_state_tracker = get_cuda_rng_tracker().get_states()
225
-
226
- # Set the states to what it used to be before the forward pass.
227
- torch.set_rng_state(ctx.fwd_cpu_rng_state)
228
- _set_cuda_rng_state(ctx.fwd_cuda_rng_state)
229
- get_cuda_rng_tracker().set_states(ctx.fwd_cuda_rng_state_tracker)
230
-
231
- # Compute the forward pass.
232
- detached_inputs = detach_variable(inputs)
233
- with torch.enable_grad():
234
- outputs = ctx.run_function(*detached_inputs)
235
-
236
- # Set the states back to what it was at the start of this function.
237
- torch.set_rng_state(bwd_cpu_rng_state)
238
- _set_cuda_rng_state(bwd_cuda_rng_state)
239
- get_cuda_rng_tracker().set_states(bwd_cuda_rng_state_tracker)
240
-
241
- if isinstance(outputs, torch.Tensor):
242
- outputs = (outputs,)
243
- torch.autograd.backward(outputs, args)
244
- grads = tuple(inp.grad if isinstance(inp, torch.Tensor) else inp
245
- for inp in detached_inputs)
246
- return (None, None) + grads
247
-
248
-
249
-def checkpoint(function, distribute_saved_activations, *args):
250
- """Checkpoint a model or part of the model.
251
- This has been directly copied from torch.utils.checkpoint."""
252
- return CheckpointFunction.apply(function,
253
- distribute_saved_activations, *args)
cacheflow/model_executor/parallel_utils/tensor_parallel/utils.py
+15
-53
@@ -1,10 +1,23 @@
1
+# Copyright 2023 The CacheFlow team.
2
+# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
3
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
5
import torch
6
from typing import List, Sequence
7
6
-from cacheflow.model_executor.parallel_utils.utils import divide
7
-from cacheflow.model_executor.parallel_utils import parallel_state
8
+def ensure_divisibility(numerator, denominator):
9
+ """Ensure that numerator is divisible by the denominator."""
10
+ assert numerator % denominator == 0, "{} is not divisible by {}".format(
11
+ numerator, denominator
12
+ )
13
+
14
+
15
+def divide(numerator, denominator):
16
+ """Ensure that numerator is divisible by the denominator and return
17
+ the division value."""
18
+ ensure_divisibility(numerator, denominator)
19
+ return numerator // denominator
20
+
21
22
def split_tensor_along_last_dim(
23
tensor: torch.Tensor,
@@ -33,57 +46,6 @@ def split_tensor_along_last_dim(
46
47
return tensor_list
48
36
-def split_tensor_into_1d_equal_chunks(tensor, new_buffer=False):
37
- """ Break a tensor into equal 1D chunks across tensor parallel ranks.
38
-
39
- Returns a Tensor or View with this rank's portion of the data.
40
-
41
- Arguments:
42
- tensor: The tensor to split
43
-
44
- Keyword Arguments:
45
- new_buffer (bool): If True, returns a new Tensor.
46
- If False, returns a view into the existing Tensor.
47
- Default is False
48
-
49
- """
50
- partition_size = torch.numel(tensor) // \
51
- parallel_state.get_tensor_model_parallel_world_size()
52
- start_index = partition_size * parallel_state.get_tensor_model_parallel_rank()
53
- end_index = start_index + partition_size
54
- if new_buffer:
55
- data = torch.empty(partition_size, dtype=tensor.dtype,
56
- device=torch.cuda.current_device(),
57
- requires_grad=False)
58
- data.copy_(tensor.view(-1)[start_index:end_index])
59
- else:
60
- data = tensor.view(-1)[start_index:end_index]
61
- return data
62
-
63
-
64
-def gather_split_1d_tensor(tensor):
65
- """ Opposite of split_tensor_into_1d_equal_chunks. Gather values from tensor
66
- model parallel ranks.
67
-
68
- Returns a new Tensor with the gathered data.
69
-
70
- Arguments:
71
- tensor: A Tensor or view of this rank's portion of the data.
72
- """
73
- numel_gathered = torch.numel(tensor) * \
74
- parallel_state.get_tensor_model_parallel_world_size()
75
- gathered = torch.empty(numel_gathered, dtype=tensor.dtype,
76
- device=torch.cuda.current_device(),
77
- requires_grad=False)
78
- # TODO: This API is experimental in pytorch (as of Feb 2022) and
79
- # this might break in future pytorch releases. We chose this API
80
- # as opposed to torch.distributed.all_gather for efficiency reasons.
81
- # This API calls directly NCCL all-gather versus the former does
82
- # internal copies and can potentially cause slow down.
83
- torch.distributed._all_gather_base(gathered, tensor,
84
- group=parallel_state.get_tensor_model_parallel_group())
85
- return gathered
86
-
49
50
class VocabUtility:
51
""" Split the vocabulary into `world_size` chunks and return the first
cacheflow/model_executor/parallel_utils/utils.py
deleted
-120
@@ -1,120 +0,0 @@
1
-"""Utility functions used throughout Megatron core"""
2
-from functools import reduce
3
-import operator
4
-
5
-import torch
6
-
7
-from cacheflow.model_executor.parallel_utils import parallel_state
8
-
9
-
10
-def ensure_divisibility(numerator, denominator):
11
- """Ensure that numerator is divisible by the denominator."""
12
- assert numerator % denominator == 0, "{} is not divisible by {}".format(
13
- numerator, denominator
14
- )
15
-
16
-
17
-def divide(numerator, denominator):
18
- """Ensure that numerator is divisible by the denominator and return
19
- the division value."""
20
- ensure_divisibility(numerator, denominator)
21
- return numerator // denominator
22
-
23
-
24
-class GlobalMemoryBuffer:
25
- """Global buffer to avoid dynamic memory allocations.
26
- Caller should ensure that buffers of the same name
27
- are not used concurrently."""
28
-
29
- def __init__(self):
30
- self.buffer = {}
31
-
32
- def get_tensor(self, tensor_shape, dtype, name):
33
- required_len = reduce(operator.mul, tensor_shape, 1)
34
- if self.buffer.get((name, dtype), None) is None or \
35
- self.buffer[(name, dtype)].numel() < required_len:
36
- self.buffer[(name, dtype)] = \
37
- torch.empty(required_len,
38
- dtype=dtype,
39
- device=torch.cuda.current_device(),
40
- requires_grad=False)
41
-
42
- return self.buffer[(name, dtype)][0:required_len].view(*tensor_shape)
43
-
44
-def _kernel_make_viewless_tensor(inp, requires_grad):
45
- '''Make a viewless tensor.
46
-
47
- View tensors have the undesirable side-affect of retaining a reference
48
- to the originally-viewed tensor, even after manually setting the '.data'
49
- field. This method creates a new tensor that links to the old tensor's
50
- data, without linking the viewed tensor, referenced via the '._base'
51
- field.
52
- '''
53
- out = torch.empty(
54
- (1,),
55
- dtype = inp.dtype,
56
- device = inp.device,
57
- requires_grad = requires_grad,
58
- )
59
- out.data = inp.data
60
- return out
61
-
62
-class MakeViewlessTensor(torch.autograd.Function):
63
- '''
64
- Autograd function to make a viewless tensor.
65
-
66
- This function should be used in cases where the computation graph needs
67
- to be propagated, but we only want a viewless tensor (e.g.,
68
- ParallelTransformer's hidden_states). Call this function by passing
69
- 'keep_graph = True' to 'make_viewless_tensor()'.
70
- '''
71
- @staticmethod
72
- def forward(ctx, inp, requires_grad):
73
- return _kernel_make_viewless_tensor(inp, requires_grad)
74
- @staticmethod
75
- def backward(ctx, grad_output):
76
- return grad_output, None
77
-
78
-def make_viewless_tensor(inp, requires_grad, keep_graph):
79
- '''
80
- Entry-point for creating viewless tensors.
81
-
82
- This method should be used, rather than calling 'MakeViewlessTensor'
83
- or '_kernel_make_viewless_tensor' directly. This method acts as a
84
- switch for determining if an autograd function or a regular method
85
- should be used to create the tensor.
86
- '''
87
-
88
- # return tensor as-is, if not a 'view'
89
- if inp._base is None:
90
- return inp
91
-
92
- # create viewless tensor
93
- if keep_graph:
94
- return MakeViewlessTensor.apply(inp, requires_grad)
95
- else:
96
- return _kernel_make_viewless_tensor(inp, requires_grad)
97
-
98
-def assert_viewless_tensor(tensor, extra_msg = None):
99
- '''Assert that a tensor is not a view (i.e., its '._base' field is
100
- not set).'''
101
- if isinstance(tensor, list):
102
- [ assert_viewless_tensor(t) for t in tensor ]
103
- return tensor
104
- if not isinstance(tensor, torch.Tensor):
105
- return tensor
106
- assert tensor._base is None, (
107
- "Ensure tensor._base is None before setting tensor.data or storing "
108
- "tensor to memory buffer. Otherwise, a memory leak will occur (and "
109
- "likely accumulate over iterations). %s"
110
- ) % extra_msg
111
- return tensor
112
-
113
-def safely_set_viewless_tensor_data(tensor, new_data_tensor):
114
- '''Safely set tensor's '.data' field.
115
-
116
- Check first that the tensor is viewless (i.e., '._base' not set). If not,
117
- raise an exception.
118
- '''
119
- assert_viewless_tensor(tensor, extra_msg = "FYI, tensor._base has shape %s, and new_data_tensor has shape %s." % ("--" if tensor._base is None else tensor._base.shape, new_data_tensor.shape))
120
- tensor.data = new_data_tensor