@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7297fa6f7c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Remove unused parts in Megatron-LM code and add copyright notice (#110)

Zhuohan Li committed May 20, 2023 at 09:11 UTC 7297fa6f7c4ce6413ee005025b312c4c9d4f5f0b
8 files changed +23 -297
cacheflow/model_executor/parallel_utils/__init__.py
-2
@@ -1,6 +1,5 @@
1 import cacheflow.model_executor.parallel_utils.parallel_state
2 import cacheflow.model_executor.parallel_utils.tensor_parallel
3 -import cacheflow.model_executor.parallel_utils.utils
3
4 # Alias parallel_state as mpu, its legacy name
5 mpu = parallel_state
@@ -8,5 +7,4 @@ mpu = parallel_state
7 __all__ = [
8 "parallel_state",
9 "tensor_parallel",
11 - "utils",
10 ]
cacheflow/model_executor/parallel_utils/parallel_state.py
+2 -25
@@ -1,3 +1,5 @@
1 +# Copyright 2023 The CacheFlow team.
2 +# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
3 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
5 """Model and data parallel groups."""
@@ -5,8 +7,6 @@
7 import torch
8 from typing import Optional
9
8 -from .utils import GlobalMemoryBuffer
9 -
10 # Intra-layer model parallel group that the current rank belongs to.
11 _TENSOR_MODEL_PARALLEL_GROUP = None
12 # Inter-layer model parallel group that the current rank belongs to.
@@ -44,9 +44,6 @@ _PIPELINE_GLOBAL_RANKS = None
44 # rank when broadcasting weights from src to all other data parallel ranks
45 _DATA_PARALLEL_GLOBAL_RANKS = None
46
47 -# Memory buffers to avoid dynamic memory allocation
48 -_GLOBAL_MEMORY_BUFFER = None
49 -
47 _ALL_REDUCE_LAUNCHER: Optional['GraphAllReduce'] = None
48
49 def initialize_model_parallel(
@@ -199,13 +196,6 @@ def initialize_model_parallel(
196 if rank in ranks:
197 _POSITION_EMBEDDING_GLOBAL_RANKS = position_embedding_ranks
198
202 - # Initialize global memory buffer
203 - # This isn't really "parallel state" but there isn't another good place to
204 - # put this. If we end up with a more generic initialization of megatron-core
205 - # we could stick it there
206 - _set_global_memory_buffer()
207 -
208 -
199 def initialize_all_reduce_launcher(
200 max_num_tokens: int,
201 hidden_size: int,
@@ -495,17 +485,6 @@ def get_data_parallel_rank():
485 """Return my rank for the data parallel group."""
486 return torch.distributed.get_rank(group=get_data_parallel_group())
487
498 -def _set_global_memory_buffer():
499 - """Initialize global buffer"""
500 - global _GLOBAL_MEMORY_BUFFER
501 - assert _GLOBAL_MEMORY_BUFFER is None, 'global memory buffer is already initialized'
502 - _GLOBAL_MEMORY_BUFFER = GlobalMemoryBuffer()
503 -
504 -def get_global_memory_buffer():
505 - """Return the global GlobalMemoryBuffer object"""
506 - assert _GLOBAL_MEMORY_BUFFER is not None, 'global memory buffer is not initialized'
507 - return _GLOBAL_MEMORY_BUFFER
508 -
488 def get_all_reduce_launcher() -> 'GraphAllReduce':
489 assert _ALL_REDUCE_LAUNCHER is not None, 'all reduce launcher is not initialized'
490 return _ALL_REDUCE_LAUNCHER
@@ -536,8 +515,6 @@ def destroy_model_parallel():
515 _MPU_TENSOR_MODEL_PARALLEL_RANK = None
516 global _MPU_PIPELINE_MODEL_PARALLEL_RANK
517 _MPU_PIPELINE_MODEL_PARALLEL_RANK = None
539 - global _GLOBAL_MEMORY_BUFFER
540 - _GLOBAL_MEMORY_BUFFER = None
518
519
520 class GraphAllReduce:
cacheflow/model_executor/parallel_utils/tensor_parallel/__init__.py
-6
@@ -17,15 +17,12 @@ from .mappings import (
17 )
18
19 from .random import (
20 - checkpoint,
20 get_cuda_rng_tracker,
21 model_parallel_cuda_manual_seed,
22 )
23
24 from .utils import (
25 split_tensor_along_last_dim,
27 - split_tensor_into_1d_equal_chunks,
28 - gather_split_1d_tensor,
26 )
27
28 __all__ = [
@@ -45,11 +42,8 @@ __all__ = [
42 "scatter_to_tensor_model_parallel_region",
43 "scatter_to_sequence_parallel_region",
44 # random.py
48 - "checkpoint",
45 "get_cuda_rng_tracker",
46 "model_parallel_cuda_manual_seed",
47 # utils.py
48 "split_tensor_along_last_dim",
53 - "split_tensor_into_1d_equal_chunks",
54 - "gather_split_1d_tensor",
49 ]
cacheflow/model_executor/parallel_utils/tensor_parallel/layers.py
+2
@@ -1,3 +1,5 @@
1 +# Copyright 2023 The CacheFlow team.
2 +# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/layers.py
3 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
5 # Parts of the code here are adapted from PyTorch
cacheflow/model_executor/parallel_utils/tensor_parallel/mappings.py
+2
@@ -1,3 +1,5 @@
1 +# Copyright 2023 The CacheFlow team.
2 +# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/mappings.py
3 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
5 import torch
cacheflow/model_executor/parallel_utils/tensor_parallel/random.py
+2 -91
@@ -1,3 +1,5 @@
1 +# Copyright 2023 The CacheFlow team.
2 +# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/random.py
3 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
5 # Parts of the code here are adapted from PyTorch
@@ -8,22 +10,11 @@ import contextlib
10 import torch
11 from torch import _C
12 from torch.cuda import _lazy_call, device as device_ctx_manager
11 -from torch.utils.checkpoint import detach_variable
13
14 from cacheflow.model_executor.parallel_utils.parallel_state import (
14 - get_data_parallel_rank,
15 - get_tensor_model_parallel_group,
15 get_tensor_model_parallel_rank,
17 - get_tensor_model_parallel_world_size,
16 )
17
20 -from .utils import (
21 - split_tensor_into_1d_equal_chunks,
22 - gather_split_1d_tensor,
23 -)
24 -
25 -from cacheflow.model_executor.parallel_utils.utils import safely_set_viewless_tensor_data
26 -
18 # Default name for the model parallel rng tracker.
19 _MODEL_PARALLEL_RNG_TRACKER_NAME = 'model-parallel-rng'
20
@@ -171,83 +162,3 @@ def model_parallel_cuda_manual_seed(seed):
162 # and model parallel state.
163 _CUDA_RNG_STATE_TRACKER.add(_MODEL_PARALLEL_RNG_TRACKER_NAME,
164 tensor_model_parallel_seed)
174 -
175 -
176 -class CheckpointFunction(torch.autograd.Function):
177 - """This function is adapted from torch.utils.checkpoint with
178 - two main changes:
179 - 1) torch.cuda.set_rng_state is replaced with `_set_cuda_rng_state`
180 - 2) the states in the model parallel tracker are also properly
181 - tracked/set/reset.
182 - """
183 - @staticmethod
184 - def forward(ctx, run_function, distribute_saved_activations, *args):
185 - ctx.run_function = run_function
186 - ctx.distribute_saved_activations \
187 - = distribute_saved_activations
188 -
189 - # Copy the rng states.
190 - ctx.fwd_cpu_rng_state = torch.get_rng_state()
191 - ctx.fwd_cuda_rng_state = torch.cuda.get_rng_state()
192 - ctx.fwd_cuda_rng_state_tracker = get_cuda_rng_tracker().get_states()
193 -
194 - with torch.no_grad():
195 - outputs = run_function(*args)
196 -
197 - # Divide hidden states across model parallel group and only keep
198 - # the chunk corresponding to the current rank.
199 - if distribute_saved_activations:
200 - ctx.input_0_shape = args[0].data.shape
201 - safely_set_viewless_tensor_data(
202 - args[0],
203 - split_tensor_into_1d_equal_chunks(args[0].data, new_buffer=True))
204 -
205 - # Store everything.
206 - ctx.save_for_backward(*args)
207 -
208 - return outputs
209 -
210 - @staticmethod
211 - def backward(ctx, *args):
212 - if not torch.autograd._is_checkpoint_valid():
213 - raise RuntimeError("Checkpointing is not compatible with .grad(), "
214 - "please use .backward() if possible")
215 - inputs = ctx.saved_tensors
216 - if ctx.distribute_saved_activations:
217 - safely_set_viewless_tensor_data(
218 - inputs[0],
219 - gather_split_1d_tensor(inputs[0].data).view(ctx.input_0_shape))
220 -
221 - # Store the current states.
222 - bwd_cpu_rng_state = torch.get_rng_state()
223 - bwd_cuda_rng_state = torch.cuda.get_rng_state()
224 - bwd_cuda_rng_state_tracker = get_cuda_rng_tracker().get_states()
225 -
226 - # Set the states to what it used to be before the forward pass.
227 - torch.set_rng_state(ctx.fwd_cpu_rng_state)
228 - _set_cuda_rng_state(ctx.fwd_cuda_rng_state)
229 - get_cuda_rng_tracker().set_states(ctx.fwd_cuda_rng_state_tracker)
230 -
231 - # Compute the forward pass.
232 - detached_inputs = detach_variable(inputs)
233 - with torch.enable_grad():
234 - outputs = ctx.run_function(*detached_inputs)
235 -
236 - # Set the states back to what it was at the start of this function.
237 - torch.set_rng_state(bwd_cpu_rng_state)
238 - _set_cuda_rng_state(bwd_cuda_rng_state)
239 - get_cuda_rng_tracker().set_states(bwd_cuda_rng_state_tracker)
240 -
241 - if isinstance(outputs, torch.Tensor):
242 - outputs = (outputs,)
243 - torch.autograd.backward(outputs, args)
244 - grads = tuple(inp.grad if isinstance(inp, torch.Tensor) else inp
245 - for inp in detached_inputs)
246 - return (None, None) + grads
247 -
248 -
249 -def checkpoint(function, distribute_saved_activations, *args):
250 - """Checkpoint a model or part of the model.
251 - This has been directly copied from torch.utils.checkpoint."""
252 - return CheckpointFunction.apply(function,
253 - distribute_saved_activations, *args)
cacheflow/model_executor/parallel_utils/tensor_parallel/utils.py
+15 -53
@@ -1,10 +1,23 @@
1 +# Copyright 2023 The CacheFlow team.
2 +# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
3 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
5 import torch
6 from typing import List, Sequence
7
6 -from cacheflow.model_executor.parallel_utils.utils import divide
7 -from cacheflow.model_executor.parallel_utils import parallel_state
8 +def ensure_divisibility(numerator, denominator):
9 + """Ensure that numerator is divisible by the denominator."""
10 + assert numerator % denominator == 0, "{} is not divisible by {}".format(
11 + numerator, denominator
12 + )
13 +
14 +
15 +def divide(numerator, denominator):
16 + """Ensure that numerator is divisible by the denominator and return
17 + the division value."""
18 + ensure_divisibility(numerator, denominator)
19 + return numerator // denominator
20 +
21
22 def split_tensor_along_last_dim(
23 tensor: torch.Tensor,
@@ -33,57 +46,6 @@ def split_tensor_along_last_dim(
46
47 return tensor_list
48
36 -def split_tensor_into_1d_equal_chunks(tensor, new_buffer=False):
37 - """ Break a tensor into equal 1D chunks across tensor parallel ranks.
38 -
39 - Returns a Tensor or View with this rank's portion of the data.
40 -
41 - Arguments:
42 - tensor: The tensor to split
43 -
44 - Keyword Arguments:
45 - new_buffer (bool): If True, returns a new Tensor.
46 - If False, returns a view into the existing Tensor.
47 - Default is False
48 -
49 - """
50 - partition_size = torch.numel(tensor) // \
51 - parallel_state.get_tensor_model_parallel_world_size()
52 - start_index = partition_size * parallel_state.get_tensor_model_parallel_rank()
53 - end_index = start_index + partition_size
54 - if new_buffer:
55 - data = torch.empty(partition_size, dtype=tensor.dtype,
56 - device=torch.cuda.current_device(),
57 - requires_grad=False)
58 - data.copy_(tensor.view(-1)[start_index:end_index])
59 - else:
60 - data = tensor.view(-1)[start_index:end_index]
61 - return data
62 -
63 -
64 -def gather_split_1d_tensor(tensor):
65 - """ Opposite of split_tensor_into_1d_equal_chunks. Gather values from tensor
66 - model parallel ranks.
67 -
68 - Returns a new Tensor with the gathered data.
69 -
70 - Arguments:
71 - tensor: A Tensor or view of this rank's portion of the data.
72 - """
73 - numel_gathered = torch.numel(tensor) * \
74 - parallel_state.get_tensor_model_parallel_world_size()
75 - gathered = torch.empty(numel_gathered, dtype=tensor.dtype,
76 - device=torch.cuda.current_device(),
77 - requires_grad=False)
78 - # TODO: This API is experimental in pytorch (as of Feb 2022) and
79 - # this might break in future pytorch releases. We chose this API
80 - # as opposed to torch.distributed.all_gather for efficiency reasons.
81 - # This API calls directly NCCL all-gather versus the former does
82 - # internal copies and can potentially cause slow down.
83 - torch.distributed._all_gather_base(gathered, tensor,
84 - group=parallel_state.get_tensor_model_parallel_group())
85 - return gathered
86 -
49
50 class VocabUtility:
51 """ Split the vocabulary into `world_size` chunks and return the first
cacheflow/model_executor/parallel_utils/utils.py deleted
-120
@@ -1,120 +0,0 @@
1 -"""Utility functions used throughout Megatron core"""
2 -from functools import reduce
3 -import operator
4 -
5 -import torch
6 -
7 -from cacheflow.model_executor.parallel_utils import parallel_state
8 -
9 -
10 -def ensure_divisibility(numerator, denominator):
11 - """Ensure that numerator is divisible by the denominator."""
12 - assert numerator % denominator == 0, "{} is not divisible by {}".format(
13 - numerator, denominator
14 - )
15 -
16 -
17 -def divide(numerator, denominator):
18 - """Ensure that numerator is divisible by the denominator and return
19 - the division value."""
20 - ensure_divisibility(numerator, denominator)
21 - return numerator // denominator
22 -
23 -
24 -class GlobalMemoryBuffer:
25 - """Global buffer to avoid dynamic memory allocations.
26 - Caller should ensure that buffers of the same name
27 - are not used concurrently."""
28 -
29 - def __init__(self):
30 - self.buffer = {}
31 -
32 - def get_tensor(self, tensor_shape, dtype, name):
33 - required_len = reduce(operator.mul, tensor_shape, 1)
34 - if self.buffer.get((name, dtype), None) is None or \
35 - self.buffer[(name, dtype)].numel() < required_len:
36 - self.buffer[(name, dtype)] = \
37 - torch.empty(required_len,
38 - dtype=dtype,
39 - device=torch.cuda.current_device(),
40 - requires_grad=False)
41 -
42 - return self.buffer[(name, dtype)][0:required_len].view(*tensor_shape)
43 -
44 -def _kernel_make_viewless_tensor(inp, requires_grad):
45 - '''Make a viewless tensor.
46 -
47 - View tensors have the undesirable side-affect of retaining a reference
48 - to the originally-viewed tensor, even after manually setting the '.data'
49 - field. This method creates a new tensor that links to the old tensor's
50 - data, without linking the viewed tensor, referenced via the '._base'
51 - field.
52 - '''
53 - out = torch.empty(
54 - (1,),
55 - dtype = inp.dtype,
56 - device = inp.device,
57 - requires_grad = requires_grad,
58 - )
59 - out.data = inp.data
60 - return out
61 -
62 -class MakeViewlessTensor(torch.autograd.Function):
63 - '''
64 - Autograd function to make a viewless tensor.
65 -
66 - This function should be used in cases where the computation graph needs
67 - to be propagated, but we only want a viewless tensor (e.g.,
68 - ParallelTransformer's hidden_states). Call this function by passing
69 - 'keep_graph = True' to 'make_viewless_tensor()'.
70 - '''
71 - @staticmethod
72 - def forward(ctx, inp, requires_grad):
73 - return _kernel_make_viewless_tensor(inp, requires_grad)
74 - @staticmethod
75 - def backward(ctx, grad_output):
76 - return grad_output, None
77 -
78 -def make_viewless_tensor(inp, requires_grad, keep_graph):
79 - '''
80 - Entry-point for creating viewless tensors.
81 -
82 - This method should be used, rather than calling 'MakeViewlessTensor'
83 - or '_kernel_make_viewless_tensor' directly. This method acts as a
84 - switch for determining if an autograd function or a regular method
85 - should be used to create the tensor.
86 - '''
87 -
88 - # return tensor as-is, if not a 'view'
89 - if inp._base is None:
90 - return inp
91 -
92 - # create viewless tensor
93 - if keep_graph:
94 - return MakeViewlessTensor.apply(inp, requires_grad)
95 - else:
96 - return _kernel_make_viewless_tensor(inp, requires_grad)
97 -
98 -def assert_viewless_tensor(tensor, extra_msg = None):
99 - '''Assert that a tensor is not a view (i.e., its '._base' field is
100 - not set).'''
101 - if isinstance(tensor, list):
102 - [ assert_viewless_tensor(t) for t in tensor ]
103 - return tensor
104 - if not isinstance(tensor, torch.Tensor):
105 - return tensor
106 - assert tensor._base is None, (
107 - "Ensure tensor._base is None before setting tensor.data or storing "
108 - "tensor to memory buffer. Otherwise, a memory leak will occur (and "
109 - "likely accumulate over iterations). %s"
110 - ) % extra_msg
111 - return tensor
112 -
113 -def safely_set_viewless_tensor_data(tensor, new_data_tensor):
114 - '''Safely set tensor's '.data' field.
115 -
116 - Check first that the tensor is viewless (i.e., '._base' not set). If not,
117 - raise an exception.
118 - '''
119 - assert_viewless_tensor(tensor, extra_msg = "FYI, tensor._base has shape %s, and new_data_tensor has shape %s." % ("--" if tensor._base is None else tensor._base.shape, new_data_tensor.shape))
120 - tensor.data = new_data_tensor