A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Support tensor parallel (#2)
Zhuohan Li committed
Mar 22, 2023 at 04:45 UTC
2f49f155858faaf82bfd076a821497e41e961658
24 files changed
+2482
-176
README.md
+2
-1
@@ -11,5 +11,6 @@ pip install -e .
11
## Run
12
13
```bash
14
-python server.py
14
+ray start --head
15
+python server.py [--tensor-parallel-size <N>]
16
```
cacheflow/models/__init__.py
-2
@@ -1,12 +1,10 @@
1
from cacheflow.models.input_metadata import InputMetadata
2
from cacheflow.models.model_utils import get_memory_analyzer
3
from cacheflow.models.model_utils import get_model
4
-from cacheflow.models.utils import set_seed
4
5
6
__all__ = [
7
'InputMetadata',
8
'get_memory_analyzer',
9
'get_model',
11
- 'set_seed',
10
]
cacheflow/models/attention.py
+1
-1
@@ -112,7 +112,7 @@ class OPTCacheFlowAttention(nn.Module):
112
output[:num_prompt_tokens],
113
query[:num_prompt_tokens],
114
key[:num_prompt_tokens],
115
- value[:num_prompt_tokens],
115
+ value[:num_prompt_tokens],
116
input_metadata.prompt_lens,
117
)
118
cacheflow/models/input_metadata.py
+5
-1
@@ -43,4 +43,8 @@ class InputMetadata:
43
f'num_generation_tokens={self.num_generation_tokens}, '
44
f'num_valid_tokens={self.num_valid_tokens}, '
45
f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
46
- f'max_context_len={self.max_context_len})')
46
+ f'max_context_len={self.max_context_len}), '
47
+ f'prompt_lens={self.prompt_lens}, '
48
+ f'slot_mapping={self.slot_mapping}, '
49
+ f'context_lens={self.context_lens}, '
50
+ f'block_tables={self.block_tables})')
cacheflow/models/memory_analyzer.py
+18
-16
@@ -31,12 +31,13 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
31
model_name: str,
32
block_size: int,
33
dtype: torch.dtype,
34
+ tensor_parallel_size: int,
35
) -> None:
36
self.model_name = model_name
37
self.block_size = block_size
38
self.dtype = dtype
39
+ self.tensor_parallel_size = tensor_parallel_size
40
39
- # TODO(woosuk): Support tensor parallelism.
41
config = AutoConfig.from_pretrained(model_name)
42
self.num_layers = config.num_hidden_layers
43
self.hidden_size = config.hidden_size
@@ -48,26 +49,25 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
49
self.max_position = config.max_position_embeddings
50
51
def _get_param_size(self) -> int:
51
- # TODO(woosuk): Support tensor parallelism.
52
- word_embedding = self.vocab_size * self.embedding_size
52
+ word_embedding = self.vocab_size * self.embedding_size // self.tensor_parallel_size
53
if self.embedding_size != self.vocab_size:
54
# Project in/out.
55
word_embedding += 2 * self.embedding_size * self.vocab_size
56
position_embedding = self.max_position * self.hidden_size
57
58
ln1 = 2 * self.hidden_size
59
- q = self.hidden_size * self.hidden_size + self.hidden_size
60
- k = self.hidden_size * self.hidden_size + self.hidden_size
61
- v = self.hidden_size * self.hidden_size + self.hidden_size
62
- out = self.hidden_size * self.hidden_size + self.hidden_size
59
+ q = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
60
+ k = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
61
+ v = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
62
+ out = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
63
mha = ln1 + q + k + v + out
64
65
ln2 = 2 * self.hidden_size
66
- ffn1 = self.hidden_size * self.ffn_size + self.ffn_size
67
- ffn2 = self.ffn_size * self.hidden_size + self.hidden_size
66
+ ffn1 = self.hidden_size * self.ffn_size // self.tensor_parallel_size + self.ffn_size
67
+ ffn2 = self.ffn_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
68
ffn = ln2 + ffn1 + ffn2
69
70
- total = (word_embedding + position_embedding +
70
+ total = (word_embedding + position_embedding +
71
self.num_layers * (mha + ffn))
72
dtype_size = get_dtype_size(self.dtype)
73
return dtype_size * total
@@ -76,15 +76,17 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
76
self,
77
max_num_batched_tokens: int,
78
) -> int:
79
- # TODO(woosuk): Support tensor parallelism.
79
# NOTE: We approxmiately calculate the maximum activation size by
81
- # 1) estimating the maximum activation tensor size during inference, and
82
- # 2) multiplying it by 4.
80
+ # estimating
81
+ # 1) the maximum activation tensor size during inference
82
+ # 2) the residual tensor size during inference
83
# Here, we assume that FlashAttention is used and
84
# thus the attention maps are never materialized in GPU DRAM.
85
- qkv = 3 * (max_num_batched_tokens * self.hidden_size)
86
- ffn = max_num_batched_tokens * self.ffn_size
87
- max_act = 4 * max(qkv, ffn)
85
+ residual = max_num_batched_tokens * self.hidden_size
86
+ qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
87
+ ffn = max_num_batched_tokens * self.ffn_size // self.tensor_parallel_size
88
+ # Double the activation size for input and output.
89
+ max_act = 2 * (max(qkv, ffn) + residual)
90
dtype_size = get_dtype_size(self.dtype)
91
return dtype_size * max_act
92
cacheflow/models/model_utils.py
+16
-6
@@ -1,7 +1,9 @@
1
from typing import Union
2
3
+import numpy as np
4
import torch
5
import torch.nn as nn
6
+from transformers import AutoConfig
7
8
from cacheflow.models.memory_analyzer import CacheFlowMemoryAnalyzer
9
from cacheflow.models.memory_analyzer import OPTMemoryAnalyzer
@@ -21,13 +23,20 @@ _MEMORY_ANALYZERS = {
23
def get_model(
24
model_name: str,
25
dtype: Union[torch.dtype, str],
26
+ path: str,
27
) -> nn.Module:
28
torch_dtype = get_torch_dtype(dtype)
26
- for model_class, hf_model in _MODELS.items():
27
- if model_class in model_name:
28
- model = hf_model.from_pretrained(
29
- model_name, torch_dtype=torch_dtype)
30
- return model.eval()
29
+ torch.set_default_dtype(torch_dtype)
30
+ config = AutoConfig.from_pretrained(model_name)
31
+ for model_class_name, model_class in _MODELS.items():
32
+ if model_class_name in model_name:
33
+ # Download model weights if it's not cached.
34
+ weights_dir = model_class.download_weights(model_name, path=path)
35
+ # Create a model instance.
36
+ model = model_class(config)
37
+ # Load the weights from the cached or downloaded files.
38
+ model.load_weights(weights_dir)
39
+ return model.eval(), torch_dtype
40
raise ValueError(f'Unsupported model name: {model_name}')
41
42
@@ -35,10 +44,11 @@ def get_memory_analyzer(
44
model_name: str,
45
block_size: int,
46
dtype: Union[torch.dtype, str],
47
+ tensor_parallel_size: int = 1,
48
) -> CacheFlowMemoryAnalyzer:
49
torch_dtype = get_torch_dtype(dtype)
50
for model_class, memory_analyzer in _MEMORY_ANALYZERS.items():
51
if model_class in model_name:
52
return memory_analyzer(
43
- model_name, block_size, torch_dtype)
53
+ model_name, block_size, torch_dtype, tensor_parallel_size)
54
raise ValueError(f'Unsupported model name: {model_name}')
cacheflow/models/opt.py
+131
-70
@@ -1,14 +1,24 @@
1
"""1D OPT model compatible with HuggingFace weights."""
2
+import os
3
+import glob
4
+import filelock
5
+from tqdm import tqdm
6
from typing import Dict, List, Optional, Tuple
7
8
+import numpy as np
9
import torch
10
from torch import nn
11
from transformers import OPTConfig
7
-from transformers import PreTrainedModel
12
+from huggingface_hub import snapshot_download
13
14
from cacheflow.models import InputMetadata
15
from cacheflow.models.attention import OPTCacheFlowAttention
16
from cacheflow.models.sample import Sampler
17
+from cacheflow.parallel_utils.parallel_state import (
18
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
19
+from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
20
+ ColumnParallelLinear,
21
+ RowParallelLinear)
22
from cacheflow.sequence import SequenceOutputs
23
24
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -36,15 +46,26 @@ class OPTAttention(nn.Module):
46
) -> None:
47
super().__init__()
48
self.embed_dim = embed_dim
39
- self.num_heads = num_heads
40
- self.head_dim = embed_dim // num_heads
49
+ tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
50
+ total_num_heads = num_heads
51
+ assert num_heads % tensor_model_parallel_world_size == 0
52
+ self.num_heads = total_num_heads // tensor_model_parallel_world_size
53
+ self.head_dim = embed_dim // total_num_heads
54
self.scaling = self.head_dim**-0.5
55
56
# TODO(woosuk): Fuse the three linear layers into one QKV linear layer.
44
- self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
45
- self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
46
- self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
47
- self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
57
+ self.k_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
58
+ gather_output=False,
59
+ perform_initialization=False)
60
+ self.v_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
61
+ gather_output=False,
62
+ perform_initialization=False)
63
+ self.q_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
64
+ gather_output=False,
65
+ perform_initialization=False)
66
+ self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
67
+ input_is_parallel=True,
68
+ perform_initialization=False)
69
70
self.attn = OPTCacheFlowAttention(scale=self.scaling)
71
@@ -55,13 +76,13 @@ class OPTAttention(nn.Module):
76
input_metadata: InputMetadata,
77
cache_event: Optional[torch.cuda.Event],
78
) -> torch.Tensor:
58
- q = self.q_proj(hidden_states)
59
- k = self.k_proj(hidden_states)
60
- v = self.v_proj(hidden_states)
79
+ q, _ = self.q_proj(hidden_states)
80
+ k, _ = self.k_proj(hidden_states)
81
+ v, _ = self.v_proj(hidden_states)
82
key_cache, value_cache = kv_cache
83
attn_output = self.attn(
84
q, k, v, key_cache, value_cache, input_metadata, cache_event)
64
- output = self.out_proj(attn_output)
85
+ output, _ = self.out_proj(attn_output)
86
return output
87
88
@@ -69,6 +90,7 @@ class OPTDecoderLayer(nn.Module):
90
91
def __init__(self, config: OPTConfig):
92
super().__init__()
93
+ self.config = config
94
self.embed_dim = config.hidden_size
95
self.self_attn = OPTAttention(
96
embed_dim=self.embed_dim,
@@ -81,9 +103,16 @@ class OPTDecoderLayer(nn.Module):
103
104
self.self_attn_layer_norm = nn.LayerNorm(
105
self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
84
- self.fc1 = nn.Linear(self.embed_dim, config.ffn_dim, bias=config.enable_bias)
85
- self.fc2 = nn.Linear(config.ffn_dim, self.embed_dim, bias=config.enable_bias)
86
- self.final_layer_norm = nn.LayerNorm(self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
106
+ self.fc1 = ColumnParallelLinear(self.embed_dim, config.ffn_dim,
107
+ bias=config.enable_bias,
108
+ gather_output=False,
109
+ perform_initialization=False)
110
+ self.fc2 = RowParallelLinear(config.ffn_dim, self.embed_dim,
111
+ bias=config.enable_bias,
112
+ input_is_parallel=True,
113
+ perform_initialization=False)
114
+ self.final_layer_norm = nn.LayerNorm(
115
+ self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
116
117
def forward(
118
self,
@@ -112,9 +141,9 @@ class OPTDecoderLayer(nn.Module):
141
# 125m, 1.7B, ..., 175B applies layer norm BEFORE attention
142
if self.do_layer_norm_before:
143
hidden_states = self.final_layer_norm(hidden_states)
115
- hidden_states = self.fc1(hidden_states)
144
+ hidden_states, _ = self.fc1(hidden_states)
145
hidden_states = self.activation_fn(hidden_states)
117
- hidden_states = self.fc2(hidden_states)
146
+ hidden_states, _ = self.fc2(hidden_states)
147
hidden_states = residual + hidden_states
148
# 350m applies layer norm AFTER attention
149
if not self.do_layer_norm_before:
@@ -122,29 +151,23 @@ class OPTDecoderLayer(nn.Module):
151
return hidden_states
152
153
125
-class OPTPreTrainedModel(PreTrainedModel):
126
- config_class = OPTConfig
127
- base_model_prefix = "model"
128
- supports_gradient_checkpointing = True
129
- _no_split_modules = ["OPTDecoderLayer"]
130
- _keys_to_ignore_on_load_unexpected = [r"decoder\.version"]
131
-
132
- def _init_weights(self, module) -> None:
133
- del module # unused
134
- return
135
-
136
-
137
-class OPTDecoder(OPTPreTrainedModel):
154
+class OPTDecoder(nn.Module):
155
156
def __init__(self, config: OPTConfig):
140
- super().__init__(config)
157
+ super().__init__()
158
+ self.config = config
159
self.padding_idx = config.pad_token_id
160
self.max_target_positions = config.max_position_embeddings
161
self.vocab_size = config.vocab_size
162
145
- self.embed_tokens = nn.Embedding(config.vocab_size, config.word_embed_proj_dim, self.padding_idx)
146
- self.embed_positions = OPTLearnedPositionalEmbedding(config.max_position_embeddings, config.hidden_size)
163
+ self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
164
+ config.word_embed_proj_dim,
165
+ perform_initialization=False)
166
+ # Positional embeddings are replicated (not sharded).
167
+ self.embed_positions = OPTLearnedPositionalEmbedding(
168
+ config.max_position_embeddings, config.hidden_size)
169
170
+ # Project out & in will be replicated if they exist.
171
if config.word_embed_proj_dim != config.hidden_size:
172
self.project_out = nn.Linear(config.hidden_size, config.word_embed_proj_dim, bias=False)
173
else:
@@ -167,9 +190,6 @@ class OPTDecoder(OPTPreTrainedModel):
190
191
self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)])
192
170
- # Initialize weights and apply final processing
171
- self.post_init()
172
-
193
def forward(
194
self,
195
input_ids: torch.LongTensor,
@@ -200,13 +220,11 @@ class OPTDecoder(OPTPreTrainedModel):
220
return hidden_states
221
222
203
-class OPTModel(OPTPreTrainedModel):
223
+class OPTModel(nn.Module):
224
225
def __init__(self, config: OPTConfig):
206
- super().__init__(config)
226
+ super().__init__()
227
self.decoder = OPTDecoder(config)
208
- # Initialize weights and apply final processing
209
- self.post_init()
228
229
def forward(
230
self,
@@ -220,41 +238,17 @@ class OPTModel(OPTPreTrainedModel):
238
input_ids, positions, kv_caches, input_metadata, cache_events)
239
240
223
-class OPTForCausalLM(OPTPreTrainedModel):
224
- _keys_to_ignore_on_load_missing = [r"lm_head.weight"]
241
+class OPTForCausalLM(nn.Module):
242
243
def __init__(self, config):
227
- super().__init__(config)
244
+ super().__init__()
245
+ self.config = config
246
self.model = OPTModel(config)
229
- # the lm_head weight is automatically tied to the embed tokens weight
230
- self.lm_head = nn.Linear(config.word_embed_proj_dim, config.vocab_size, bias=False)
247
+ # TODO(zhuohan): create a new weight after implementing pipeline
248
+ # parallelism
249
+ self.lm_head_weight = self.model.decoder.embed_tokens.weight
250
self.sampler = Sampler()
251
233
- # Initialize weights and apply final processing
234
- self.post_init()
235
-
236
- # NOTE(woosuk): While the following methods are not called in the model code,
237
- # they may be internally used by the transformers library.
238
- # For example, tie_weights() does not work without these methods.
239
- # Thus, do not delete these methods.
240
- def get_input_embeddings(self):
241
- return self.model.decoder.embed_tokens
242
-
243
- def set_input_embeddings(self, value):
244
- self.model.decoder.embed_tokens = value
245
-
246
- def get_output_embeddings(self):
247
- return self.lm_head
248
-
249
- def set_output_embeddings(self, new_embeddings):
250
- self.lm_head = new_embeddings
251
-
252
- def set_decoder(self, decoder):
253
- self.model.decoder = decoder
254
-
255
- def get_decoder(self):
256
- return self.model.decoder
257
-
252
def forward(
253
self,
254
input_ids: torch.LongTensor,
@@ -266,5 +260,72 @@ class OPTForCausalLM(OPTPreTrainedModel):
260
hidden_states = self.model(
261
input_ids, positions, kv_caches, input_metadata, cache_events)
262
next_tokens = self.sampler(
269
- self.lm_head.weight, hidden_states, input_metadata)
263
+ self.lm_head_weight, hidden_states, input_metadata)
264
return next_tokens
265
+
266
+ _column_parallel_weights = ["embed_tokens.weight",
267
+ "q_proj.weight", "k_proj.weight",
268
+ "v_proj.weight", "fc1.weight"]
269
+ _column_parallel_biases = ["q_proj.bias", "k_proj.bias",
270
+ "v_proj.bias", "fc1.bias"]
271
+ _row_parallel_weights = ["out_proj.weight", "fc2.weight"]
272
+
273
+ def load_weights(self, weights_path: str):
274
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
275
+ state_dict = self.state_dict()
276
+ for name, param in state_dict.items():
277
+ if "lm_head_weight" in name:
278
+ continue
279
+ loaded_weight = torch.from_numpy(np.load(os.path.join(weights_path,
280
+ name)))
281
+ for p in (self._column_parallel_weights
282
+ + self._column_parallel_biases):
283
+ if p in name:
284
+ shard_size = param.shape[0]
285
+ loaded_weight = loaded_weight[
286
+ shard_size * tensor_model_parallel_rank
287
+ :shard_size * (tensor_model_parallel_rank + 1)]
288
+ break
289
+ for p in self._row_parallel_weights:
290
+ if p in name:
291
+ shard_size = param.shape[1]
292
+ loaded_weight = loaded_weight[
293
+ :,
294
+ shard_size * tensor_model_parallel_rank
295
+ :shard_size * (tensor_model_parallel_rank + 1)]
296
+ break
297
+
298
+ assert param.shape == loaded_weight.shape
299
+ param.data.copy_(loaded_weight)
300
+
301
+ @staticmethod
302
+ def download_weights(model_name: str, path: str):
303
+ path = os.path.join(path, f"{model_name}-np")
304
+ path = os.path.abspath(os.path.expanduser(path))
305
+ os.makedirs(path, exist_ok=True)
306
+ lock_path = os.path.join(path, "file_lock")
307
+ lock = filelock.FileLock(lock_path)
308
+
309
+ with lock:
310
+ test_weight_path = os.path.join(
311
+ path, "model.decoder.embed_positions.weight")
312
+ if os.path.exists(test_weight_path):
313
+ return path
314
+
315
+ folder = snapshot_download(model_name, allow_patterns="*.bin",
316
+ cache_dir=os.path.join(path, "cache"))
317
+ bin_files = glob.glob(os.path.join(folder, "*.bin"))
318
+
319
+ if "/" in model_name:
320
+ model_name = model_name.split("/")[1].lower()
321
+
322
+ for bin_file in tqdm(bin_files, desc="Convert format"):
323
+ state = torch.load(bin_file)
324
+ for name, param in tqdm(state.items(), leave=False):
325
+ if name.startswith("decoder."):
326
+ name = "model." + name
327
+ param_path = os.path.join(path, name)
328
+ with open(param_path, "wb") as f:
329
+ np.save(f, param.cpu().detach().numpy())
330
+
331
+ return path
cacheflow/models/sample.py
+2
-1
@@ -6,7 +6,7 @@ import torch.nn as nn
6
from cacheflow.models import InputMetadata
7
from cacheflow.sampling_params import SamplingParams
8
from cacheflow.sequence import SequenceOutputs
9
-
9
+from cacheflow.parallel_utils.tensor_parallel import gather_from_tensor_model_parallel_region
10
11
class Sampler(nn.Module):
12
@@ -24,6 +24,7 @@ class Sampler(nn.Module):
24
25
# Get the logits for the next tokens.
26
logits = torch.matmul(hidden_states, embedding.t())
27
+ logits = gather_from_tensor_model_parallel_region(logits)
28
29
# Apply temperature scaling.
30
temperatures = _get_temperatures(input_metadata)
cacheflow/models/utils.py
-8
@@ -27,14 +27,6 @@ def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
27
return torch.tensor([], dtype=torch_dtype).element_size()
28
29
30
-def set_seed(seed: int) -> None:
31
- random.seed(seed)
32
- np.random.seed(seed)
33
- torch.manual_seed(seed)
34
- if torch.cuda.is_available():
35
- torch.cuda.manual_seed_all(seed)
36
-
37
-
30
def get_gpu_memory(gpu: int = 0) -> int:
31
return torch.cuda.get_device_properties(gpu).total_memory
32
cacheflow/parallel_utils/README.md
new
+1
@@ -0,0 +1 @@
1
+The files in this folder are ported from [Megatron-LM](https://github.com/NVIDIA/Megatron-LM/tree/main/megatron/core). We only keep the codes that are used in inference.
\ No newline at end of file
cacheflow/parallel_utils/__init__.py
new
+12
@@ -0,0 +1,12 @@
1
+import cacheflow.parallel_utils.parallel_state
2
+import cacheflow.parallel_utils.tensor_parallel
3
+import cacheflow.parallel_utils.utils
4
+
5
+# Alias parallel_state as mpu, its legacy name
6
+mpu = parallel_state
7
+
8
+__all__ = [
9
+ "parallel_state",
10
+ "tensor_parallel",
11
+ "utils",
12
+]
cacheflow/parallel_utils/parallel_state.py
new
+522
@@ -0,0 +1,522 @@
1
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2
+
3
+"""Model and data parallel groups."""
4
+
5
+import torch
6
+from typing import Optional
7
+
8
+from .utils import GlobalMemoryBuffer
9
+
10
+# Intra-layer model parallel group that the current rank belongs to.
11
+_TENSOR_MODEL_PARALLEL_GROUP = None
12
+# Inter-layer model parallel group that the current rank belongs to.
13
+_PIPELINE_MODEL_PARALLEL_GROUP = None
14
+# Model parallel group (both intra- and pipeline) that the current rank belongs to.
15
+_MODEL_PARALLEL_GROUP = None
16
+# Embedding group.
17
+_EMBEDDING_GROUP = None
18
+# Position embedding group.
19
+_POSITION_EMBEDDING_GROUP = None
20
+# Data parallel group that the current rank belongs to.
21
+_DATA_PARALLEL_GROUP = None
22
+
23
+_VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = None
24
+_VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
25
+_PIPELINE_MODEL_PARALLEL_SPLIT_RANK = None
26
+
27
+# These values enable us to change the mpu sizes on the fly.
28
+_MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = None
29
+_MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
30
+_MPU_TENSOR_MODEL_PARALLEL_RANK = None
31
+_MPU_PIPELINE_MODEL_PARALLEL_RANK = None
32
+
33
+# A list of ranks that have a copy of the embedding.
34
+_EMBEDDING_GLOBAL_RANKS = None
35
+
36
+# A list of ranks that have a copy of the position embedding.
37
+_POSITION_EMBEDDING_GLOBAL_RANKS = None
38
+
39
+# A list of global ranks for each pipeline group to ease calculation of the source
40
+# rank when broadcasting from the first or last pipeline stage.
41
+_PIPELINE_GLOBAL_RANKS = None
42
+
43
+# A list of global ranks for each data parallel group to ease calculation of the source
44
+# rank when broadcasting weights from src to all other data parallel ranks
45
+_DATA_PARALLEL_GLOBAL_RANKS = None
46
+
47
+# Memory buffers to avoid dynamic memory allocation
48
+_GLOBAL_MEMORY_BUFFER = None
49
+
50
+
51
+def initialize_model_parallel(
52
+ tensor_model_parallel_size: int = 1,
53
+ pipeline_model_parallel_size: int = 1,
54
+ virtual_pipeline_model_parallel_size: Optional[int] = None,
55
+ pipeline_model_parallel_split_rank: Optional[int] = None,
56
+) -> None:
57
+ """
58
+ Initialize model data parallel groups.
59
+
60
+ Arguments:
61
+ tensor_model_parallel_size: number of GPUs used for tensor model parallelism.
62
+ pipeline_model_parallel_size: number of GPUs used for pipeline model parallelism.
63
+ virtual_pipeline_model_parallel_size: number of virtual stages (interleaved
64
+ pipeline).
65
+ pipeline_model_parallel_split_rank: for models with both encoder and decoder,
66
+ rank in pipeline with split point.
67
+
68
+ Let's say we have a total of 16 GPUs denoted by g0 ... g15 and we
69
+ use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize
70
+ the model pipeline. The present function will
71
+ create 8 tensor model-parallel groups, 4 pipeline model-parallel groups
72
+ and 8 data-parallel groups as:
73
+ 8 data_parallel groups:
74
+ [g0, g2], [g1, g3], [g4, g6], [g5, g7], [g8, g10], [g9, g11], [g12, g14], [g13, g15]
75
+ 8 tensor model-parallel groups:
76
+ [g0, g1], [g2, g3], [g4, g5], [g6, g7], [g8, g9], [g10, g11], [g12, g13], [g14, g15]
77
+ 4 pipeline model-parallel groups:
78
+ [g0, g4, g8, g12], [g1, g5, g9, g13], [g2, g6, g10, g14], [g3, g7, g11, g15]
79
+ Note that for efficiency, the caller should make sure adjacent ranks
80
+ are on the same DGX box. For example if we are using 2 DGX-1 boxes
81
+ with a total of 16 GPUs, rank 0 to 7 belong to the first box and
82
+ ranks 8 to 15 belong to the second box.
83
+ """
84
+ # Get world size and rank. Ensure some consistencies.
85
+ assert torch.distributed.is_initialized()
86
+ world_size: int = torch.distributed.get_world_size()
87
+
88
+ if world_size % (tensor_model_parallel_size * pipeline_model_parallel_size) != 0:
89
+ raise RuntimeError(
90
+ f"world_size ({world_size}) is not divisible by tensor_model_parallel_size "
91
+ f"({tensor_model_parallel_size}) x pipeline_model_parallel_size ({pipeline_model_parallel_size})"
92
+ )
93
+
94
+ data_parallel_size: int = world_size // (tensor_model_parallel_size *
95
+ pipeline_model_parallel_size)
96
+
97
+ num_tensor_model_parallel_groups: int = world_size // tensor_model_parallel_size
98
+ num_pipeline_model_parallel_groups: int = world_size // pipeline_model_parallel_size
99
+ num_data_parallel_groups: int = world_size // data_parallel_size
100
+
101
+ if virtual_pipeline_model_parallel_size is not None:
102
+ if not pipeline_model_parallel_size > 2:
103
+ raise RuntimeError("pipeline-model-parallel size should be greater than 2 with "
104
+ "interleaved schedule")
105
+ global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
106
+ global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
107
+ _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = 0
108
+ _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = virtual_pipeline_model_parallel_size
109
+
110
+ if pipeline_model_parallel_split_rank is not None:
111
+ global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK
112
+ _PIPELINE_MODEL_PARALLEL_SPLIT_RANK = pipeline_model_parallel_split_rank
113
+
114
+ rank = torch.distributed.get_rank()
115
+
116
+ # Build the data-parallel groups.
117
+ global _DATA_PARALLEL_GROUP
118
+ global _DATA_PARALLEL_GLOBAL_RANKS
119
+ assert _DATA_PARALLEL_GROUP is None, 'data parallel group is already initialized'
120
+ all_data_parallel_group_ranks = []
121
+ for i in range(pipeline_model_parallel_size):
122
+ start_rank = i * num_pipeline_model_parallel_groups
123
+ end_rank = (i + 1) * num_pipeline_model_parallel_groups
124
+ for j in range(tensor_model_parallel_size):
125
+ ranks = range(start_rank + j, end_rank, tensor_model_parallel_size)
126
+ all_data_parallel_group_ranks.append(list(ranks))
127
+ group = torch.distributed.new_group(ranks)
128
+ if rank in ranks:
129
+ _DATA_PARALLEL_GROUP = group
130
+ _DATA_PARALLEL_GLOBAL_RANKS = ranks
131
+
132
+ # Build the model-parallel groups.
133
+ global _MODEL_PARALLEL_GROUP
134
+ assert _MODEL_PARALLEL_GROUP is None, 'model parallel group is already initialized'
135
+ for i in range(data_parallel_size):
136
+ ranks = [data_parallel_group_ranks[i]
137
+ for data_parallel_group_ranks in all_data_parallel_group_ranks]
138
+ group = torch.distributed.new_group(ranks)
139
+ if rank in ranks:
140
+ _MODEL_PARALLEL_GROUP = group
141
+
142
+ # Build the tensor model-parallel groups.
143
+ global _TENSOR_MODEL_PARALLEL_GROUP
144
+ assert _TENSOR_MODEL_PARALLEL_GROUP is None, \
145
+ 'tensor model parallel group is already initialized'
146
+ for i in range(num_tensor_model_parallel_groups):
147
+ ranks = range(i * tensor_model_parallel_size,
148
+ (i + 1) * tensor_model_parallel_size)
149
+ group = torch.distributed.new_group(ranks)
150
+ if rank in ranks:
151
+ _TENSOR_MODEL_PARALLEL_GROUP = group
152
+
153
+ # Build the pipeline model-parallel groups and embedding groups
154
+ # (first and last rank in each pipeline model-parallel group).
155
+ global _PIPELINE_MODEL_PARALLEL_GROUP
156
+ global _PIPELINE_GLOBAL_RANKS
157
+ assert _PIPELINE_MODEL_PARALLEL_GROUP is None, \
158
+ 'pipeline model parallel group is already initialized'
159
+ global _EMBEDDING_GROUP
160
+ global _EMBEDDING_GLOBAL_RANKS
161
+ assert _EMBEDDING_GROUP is None, 'embedding group is already initialized'
162
+ global _POSITION_EMBEDDING_GROUP
163
+ global _POSITION_EMBEDDING_GLOBAL_RANKS
164
+ assert _POSITION_EMBEDDING_GROUP is None, \
165
+ 'position embedding group is already initialized'
166
+ for i in range(num_pipeline_model_parallel_groups):
167
+ ranks = range(i, world_size, num_pipeline_model_parallel_groups)
168
+ group = torch.distributed.new_group(ranks)
169
+ if rank in ranks:
170
+ _PIPELINE_MODEL_PARALLEL_GROUP = group
171
+ _PIPELINE_GLOBAL_RANKS = ranks
172
+ # Setup embedding group (to exchange gradients between
173
+ # first and last stages).
174
+ if len(ranks) > 1:
175
+ embedding_ranks = [ranks[0], ranks[-1]]
176
+ position_embedding_ranks = [ranks[0]]
177
+ if pipeline_model_parallel_split_rank is not None:
178
+ if ranks[pipeline_model_parallel_split_rank] not in embedding_ranks:
179
+ embedding_ranks = [ranks[0],
180
+ ranks[pipeline_model_parallel_split_rank],
181
+ ranks[-1]]
182
+ if ranks[pipeline_model_parallel_split_rank] not in position_embedding_ranks:
183
+ position_embedding_ranks = [ranks[0],
184
+ ranks[pipeline_model_parallel_split_rank]]
185
+ else:
186
+ embedding_ranks = ranks
187
+ position_embedding_ranks = ranks
188
+
189
+ group = torch.distributed.new_group(embedding_ranks)
190
+ if rank in embedding_ranks:
191
+ _EMBEDDING_GROUP = group
192
+ if rank in ranks:
193
+ _EMBEDDING_GLOBAL_RANKS = embedding_ranks
194
+
195
+ group = torch.distributed.new_group(position_embedding_ranks)
196
+ if rank in position_embedding_ranks:
197
+ _POSITION_EMBEDDING_GROUP = group
198
+ if rank in ranks:
199
+ _POSITION_EMBEDDING_GLOBAL_RANKS = position_embedding_ranks
200
+
201
+ # Initialize global memory buffer
202
+ # This isn't really "parallel state" but there isn't another good place to
203
+ # put this. If we end up with a more generic initialization of megatron-core
204
+ # we could stick it there
205
+ _set_global_memory_buffer()
206
+
207
+
208
+def model_parallel_is_initialized():
209
+ """Check if model and data parallel groups are initialized."""
210
+ if _TENSOR_MODEL_PARALLEL_GROUP is None or \
211
+ _PIPELINE_MODEL_PARALLEL_GROUP is None or \
212
+ _DATA_PARALLEL_GROUP is None:
213
+ return False
214
+ return True
215
+
216
+
217
+def get_model_parallel_group():
218
+ """Get the model parallel group the caller rank belongs to."""
219
+ assert _MODEL_PARALLEL_GROUP is not None, \
220
+ 'model parallel group is not initialized'
221
+ return _MODEL_PARALLEL_GROUP
222
+
223
+
224
+def get_tensor_model_parallel_group():
225
+ """Get the tensor model parallel group the caller rank belongs to."""
226
+ assert _TENSOR_MODEL_PARALLEL_GROUP is not None, \
227
+ 'intra_layer_model parallel group is not initialized'
228
+ return _TENSOR_MODEL_PARALLEL_GROUP
229
+
230
+
231
+def get_pipeline_model_parallel_group():
232
+ """Get the pipeline model parallel group the caller rank belongs to."""
233
+ assert _PIPELINE_MODEL_PARALLEL_GROUP is not None, \
234
+ 'pipeline_model parallel group is not initialized'
235
+ return _PIPELINE_MODEL_PARALLEL_GROUP
236
+
237
+
238
+def get_data_parallel_group():
239
+ """Get the data parallel group the caller rank belongs to."""
240
+ assert _DATA_PARALLEL_GROUP is not None, \
241
+ 'data parallel group is not initialized'
242
+ return _DATA_PARALLEL_GROUP
243
+
244
+
245
+def get_embedding_group():
246
+ """Get the embedding group the caller rank belongs to."""
247
+ assert _EMBEDDING_GROUP is not None, \
248
+ 'embedding group is not initialized'
249
+ return _EMBEDDING_GROUP
250
+
251
+
252
+def get_position_embedding_group():
253
+ """Get the position embedding group the caller rank belongs to."""
254
+ assert _POSITION_EMBEDDING_GROUP is not None, \
255
+ 'position embedding group is not initialized'
256
+ return _POSITION_EMBEDDING_GROUP
257
+
258
+
259
+def set_tensor_model_parallel_world_size(world_size):
260
+ """Set the tensor model parallel size"""
261
+ global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
262
+ _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = world_size
263
+
264
+
265
+def set_pipeline_model_parallel_world_size(world_size):
266
+ """Set the pipeline model parallel size"""
267
+ global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
268
+ _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = world_size
269
+
270
+
271
+def get_tensor_model_parallel_world_size():
272
+ """Return world size for the tensor model parallel group."""
273
+ global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
274
+ if _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE is not None:
275
+ return _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
276
+ return torch.distributed.get_world_size(group=get_tensor_model_parallel_group())
277
+
278
+
279
+def get_pipeline_model_parallel_world_size():
280
+ """Return world size for the pipeline model parallel group."""
281
+ global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
282
+ if _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE is not None:
283
+ return _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
284
+ return torch.distributed.get_world_size(group=get_pipeline_model_parallel_group())
285
+
286
+
287
+def set_tensor_model_parallel_rank(rank):
288
+ """Set tensor model parallel rank."""
289
+ global _MPU_TENSOR_MODEL_PARALLEL_RANK
290
+ _MPU_TENSOR_MODEL_PARALLEL_RANK = rank
291
+
292
+
293
+def set_pipeline_model_parallel_rank(rank):
294
+ """Set pipeline model parallel rank."""
295
+ global _MPU_PIPELINE_MODEL_PARALLEL_RANK
296
+ _MPU_PIPELINE_MODEL_PARALLEL_RANK = rank
297
+
298
+
299
+def set_pipeline_model_parallel_split_rank(rank):
300
+ """Set pipeline model parallel split rank."""
301
+ global _MPU_PIPELINE_MODEL_PARALLEL_SPLIT_RANK
302
+ _MPU_PIPELINE_MODEL_PARALLEL_SPLIT_RANK = rank
303
+
304
+
305
+def get_tensor_model_parallel_rank():
306
+ """Return my rank for the tensor model parallel group."""
307
+ global _MPU_TENSOR_MODEL_PARALLEL_RANK
308
+ if _MPU_TENSOR_MODEL_PARALLEL_RANK is not None:
309
+ return _MPU_TENSOR_MODEL_PARALLEL_RANK
310
+ return torch.distributed.get_rank(group=get_tensor_model_parallel_group())
311
+
312
+
313
+def get_pipeline_model_parallel_rank():
314
+ """Return my rank for the pipeline model parallel group."""
315
+ global _MPU_PIPELINE_MODEL_PARALLEL_RANK
316
+ if _MPU_PIPELINE_MODEL_PARALLEL_RANK is not None:
317
+ return _MPU_PIPELINE_MODEL_PARALLEL_RANK
318
+ return torch.distributed.get_rank(group=get_pipeline_model_parallel_group())
319
+
320
+
321
+
322
+def is_pipeline_first_stage(ignore_virtual=False):
323
+ """Return True if in the first pipeline model-parallel stage, False otherwise."""
324
+ if not ignore_virtual:
325
+ if get_virtual_pipeline_model_parallel_world_size() is not None and \
326
+ get_virtual_pipeline_model_parallel_rank() != 0:
327
+ return False
328
+ return get_pipeline_model_parallel_rank() == 0
329
+
330
+
331
+def is_pipeline_last_stage(ignore_virtual=False):
332
+ """Return True if in the last pipeline model-parallel stage, False otherwise."""
333
+ if not ignore_virtual:
334
+ virtual_pipeline_model_parallel_world_size = \
335
+ get_virtual_pipeline_model_parallel_world_size()
336
+ if virtual_pipeline_model_parallel_world_size is not None and \
337
+ get_virtual_pipeline_model_parallel_rank() != (
338
+ virtual_pipeline_model_parallel_world_size - 1):
339
+ return False
340
+ return get_pipeline_model_parallel_rank() == (
341
+ get_pipeline_model_parallel_world_size() - 1)
342
+
343
+
344
+def is_rank_in_embedding_group(ignore_virtual=False):
345
+ """Return true if current rank is in embedding group, False otherwise."""
346
+ rank = torch.distributed.get_rank()
347
+ global _EMBEDDING_GLOBAL_RANKS
348
+ if ignore_virtual:
349
+ return rank in _EMBEDDING_GLOBAL_RANKS
350
+ if rank in _EMBEDDING_GLOBAL_RANKS:
351
+ if rank == _EMBEDDING_GLOBAL_RANKS[0]:
352
+ return is_pipeline_first_stage(ignore_virtual=False)
353
+ elif rank == _EMBEDDING_GLOBAL_RANKS[-1]:
354
+ return is_pipeline_last_stage(ignore_virtual=False)
355
+ else:
356
+ return True
357
+ return False
358
+
359
+
360
+def is_rank_in_position_embedding_group():
361
+ """Return true if current rank is in position embedding group, False otherwise."""
362
+ rank = torch.distributed.get_rank()
363
+ global _POSITION_EMBEDDING_GLOBAL_RANKS
364
+ return rank in _POSITION_EMBEDDING_GLOBAL_RANKS
365
+
366
+
367
+def is_pipeline_stage_before_split(rank=None):
368
+ """Return True if pipeline stage executes encoder block for a model
369
+ with both encoder and decoder."""
370
+ if get_pipeline_model_parallel_world_size() == 1:
371
+ return True
372
+ if rank is None:
373
+ rank = get_pipeline_model_parallel_rank()
374
+ global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK
375
+ if _PIPELINE_MODEL_PARALLEL_SPLIT_RANK is None:
376
+ return True
377
+ if rank < _PIPELINE_MODEL_PARALLEL_SPLIT_RANK:
378
+ return True
379
+ return False
380
+
381
+
382
+def is_pipeline_stage_after_split(rank=None):
383
+ """Return True if pipeline stage executes decoder block for a model
384
+ with both encoder and decoder."""
385
+ if get_pipeline_model_parallel_world_size() == 1:
386
+ return True
387
+ if rank is None:
388
+ rank = get_pipeline_model_parallel_rank()
389
+ global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK
390
+ if _PIPELINE_MODEL_PARALLEL_SPLIT_RANK is None:
391
+ return True
392
+ if rank >= _PIPELINE_MODEL_PARALLEL_SPLIT_RANK:
393
+ return True
394
+ return False
395
+
396
+
397
+def is_pipeline_stage_at_split():
398
+ """Return true if pipeline stage executes decoder block and next
399
+ stage executes encoder block for a model with both encoder and
400
+ decoder."""
401
+ rank = get_pipeline_model_parallel_rank()
402
+ return is_pipeline_stage_before_split(rank) and \
403
+ is_pipeline_stage_after_split(rank+1)
404
+
405
+
406
+def get_virtual_pipeline_model_parallel_rank():
407
+ """Return the virtual pipeline-parallel rank."""
408
+ global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
409
+ return _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
410
+
411
+
412
+def set_virtual_pipeline_model_parallel_rank(rank):
413
+ """Set the virtual pipeline-parallel rank."""
414
+ global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
415
+ _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = rank
416
+
417
+
418
+def get_virtual_pipeline_model_parallel_world_size():
419
+ """Return the virtual pipeline-parallel world size."""
420
+ global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
421
+ return _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
422
+
423
+
424
+def get_tensor_model_parallel_src_rank():
425
+ """Calculate the global rank corresponding to the first local rank
426
+ in the tensor model parallel group."""
427
+ global_rank = torch.distributed.get_rank()
428
+ local_world_size = get_tensor_model_parallel_world_size()
429
+ return (global_rank // local_world_size) * local_world_size
430
+
431
+
432
+def get_data_parallel_src_rank():
433
+ """Calculate the global rank corresponding to the first local rank
434
+ in the data parallel group."""
435
+ assert _DATA_PARALLEL_GLOBAL_RANKS is not None, \
436
+ "Data parallel group is not initialized"
437
+ return _DATA_PARALLEL_GLOBAL_RANKS[0]
438
+
439
+
440
+def get_pipeline_model_parallel_first_rank():
441
+ """Return the global rank of the first process in the pipeline for the
442
+ current tensor parallel group"""
443
+ assert _PIPELINE_GLOBAL_RANKS is not None, \
444
+ "Pipeline parallel group is not initialized"
445
+ return _PIPELINE_GLOBAL_RANKS[0]
446
+
447
+
448
+def get_pipeline_model_parallel_last_rank():
449
+ """Return the global rank of the last process in the pipeline for the
450
+ current tensor parallel group"""
451
+ assert _PIPELINE_GLOBAL_RANKS is not None, \
452
+ "Pipeline parallel group is not initialized"
453
+ last_rank_local = get_pipeline_model_parallel_world_size() - 1
454
+ return _PIPELINE_GLOBAL_RANKS[last_rank_local]
455
+
456
+def get_pipeline_model_parallel_next_rank():
457
+ """Return the global rank that follows the caller in the pipeline"""
458
+ assert _PIPELINE_GLOBAL_RANKS is not None, \
459
+ "Pipeline parallel group is not initialized"
460
+ rank_in_pipeline = get_pipeline_model_parallel_rank()
461
+ world_size = get_pipeline_model_parallel_world_size()
462
+ return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline + 1) % world_size]
463
+
464
+
465
+def get_pipeline_model_parallel_prev_rank():
466
+ """Return the global rank that preceeds the caller in the pipeline"""
467
+ assert _PIPELINE_GLOBAL_RANKS is not None, \
468
+ "Pipeline parallel group is not initialized"
469
+ rank_in_pipeline = get_pipeline_model_parallel_rank()
470
+ world_size = get_pipeline_model_parallel_world_size()
471
+ return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline - 1) % world_size]
472
+
473
+
474
+def get_data_parallel_world_size():
475
+ """Return world size for the data parallel group."""
476
+ return torch.distributed.get_world_size(group=get_data_parallel_group())
477
+
478
+
479
+def get_data_parallel_rank():
480
+ """Return my rank for the data parallel group."""
481
+ return torch.distributed.get_rank(group=get_data_parallel_group())
482
+
483
+def _set_global_memory_buffer():
484
+ """Initialize global buffer"""
485
+ global _GLOBAL_MEMORY_BUFFER
486
+ assert _GLOBAL_MEMORY_BUFFER is None, 'global memory buffer is already initialized'
487
+ _GLOBAL_MEMORY_BUFFER = GlobalMemoryBuffer()
488
+
489
+def get_global_memory_buffer():
490
+ """Return the global GlobalMemoryBuffer object"""
491
+ assert _GLOBAL_MEMORY_BUFFER is not None, 'global memory buffer is not initialized'
492
+ return _GLOBAL_MEMORY_BUFFER
493
+
494
+
495
+def destroy_model_parallel():
496
+ """Set the groups to none."""
497
+ global _MODEL_PARALLEL_GROUP
498
+ _MODEL_PARALLEL_GROUP = None
499
+ global _TENSOR_MODEL_PARALLEL_GROUP
500
+ _TENSOR_MODEL_PARALLEL_GROUP = None
501
+ global _PIPELINE_MODEL_PARALLEL_GROUP
502
+ _PIPELINE_MODEL_PARALLEL_GROUP = None
503
+ global _DATA_PARALLEL_GROUP
504
+ _DATA_PARALLEL_GROUP = None
505
+ global _EMBEDDING_GROUP
506
+ _EMBEDDING_GROUP = None
507
+ global _POSITION_EMBEDDING_GROUP
508
+ _POSITION_EMBEDDING_GROUP = None
509
+ global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
510
+ _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = None
511
+ global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
512
+ _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
513
+ global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
514
+ _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = None
515
+ global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
516
+ _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
517
+ global _MPU_TENSOR_MODEL_PARALLEL_RANK
518
+ _MPU_TENSOR_MODEL_PARALLEL_RANK = None
519
+ global _MPU_PIPELINE_MODEL_PARALLEL_RANK
520
+ _MPU_PIPELINE_MODEL_PARALLEL_RANK = None
521
+ global _GLOBAL_MEMORY_BUFFER
522
+ _GLOBAL_MEMORY_BUFFER = None
cacheflow/parallel_utils/tensor_parallel/__init__.py
new
+58
@@ -0,0 +1,58 @@
1
+from .layers import (
2
+ ColumnParallelLinear,
3
+ RowParallelLinear,
4
+ VocabParallelEmbedding,
5
+ set_tensor_model_parallel_attributes,
6
+ set_defaults_if_not_set_tensor_model_parallel_attributes,
7
+ copy_tensor_model_parallel_attributes,
8
+ param_is_not_tensor_parallel_duplicate,
9
+ linear_with_grad_accumulation_and_async_allreduce
10
+
11
+)
12
+
13
+from .mappings import (
14
+ copy_to_tensor_model_parallel_region,
15
+ gather_from_tensor_model_parallel_region,
16
+ gather_from_sequence_parallel_region,
17
+ scatter_to_tensor_model_parallel_region,
18
+ scatter_to_sequence_parallel_region,
19
+)
20
+
21
+from .random import (
22
+ checkpoint,
23
+ get_cuda_rng_tracker,
24
+ model_parallel_cuda_manual_seed,
25
+)
26
+
27
+from .utils import (
28
+ split_tensor_along_last_dim,
29
+ split_tensor_into_1d_equal_chunks,
30
+ gather_split_1d_tensor,
31
+)
32
+
33
+__all__ = [
34
+ #layers.py
35
+ "ColumnParallelLinear",
36
+ "RowParallelLinear",
37
+ "VocabParallelEmbedding",
38
+ "set_tensor_model_parallel_attributes",
39
+ "set_defaults_if_not_set_tensor_model_parallel_attributes",
40
+ "copy_tensor_model_parallel_attributes",
41
+ "param_is_not_tensor_parallel_duplicate",
42
+ "linear_with_grad_accumulation_and_async_allreduce",
43
+ # mappings.py
44
+ "copy_to_tensor_model_parallel_region",
45
+ "gather_from_tensor_model_parallel_region",
46
+ "gather_from_sequence_parallel_region",
47
+# "reduce_from_tensor_model_parallel_region",
48
+ "scatter_to_tensor_model_parallel_region",
49
+ "scatter_to_sequence_parallel_region",
50
+ # random.py
51
+ "checkpoint",
52
+ "get_cuda_rng_tracker",
53
+ "model_parallel_cuda_manual_seed",
54
+ # utils.py
55
+ "split_tensor_along_last_dim",
56
+ "split_tensor_into_1d_equal_chunks",
57
+ "gather_split_1d_tensor",
58
+]
cacheflow/parallel_utils/tensor_parallel/layers.py
new
+719
@@ -0,0 +1,719 @@
1
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2
+
3
+# Parts of the code here are adapted from PyTorch
4
+# repo: https://github.com/pytorch/pytorch
5
+
6
+import math
7
+import os
8
+from typing import Optional
9
+import warnings
10
+
11
+import torch
12
+import torch.nn.functional as F
13
+import torch.nn.init as init
14
+from torch.nn.parameter import Parameter
15
+
16
+from cacheflow.parallel_utils.parallel_state import (
17
+ get_tensor_model_parallel_rank,
18
+ get_tensor_model_parallel_world_size,
19
+ get_tensor_model_parallel_group,
20
+ get_global_memory_buffer,
21
+)
22
+from .mappings import (
23
+ copy_to_tensor_model_parallel_region,
24
+ gather_from_tensor_model_parallel_region,
25
+ gather_from_sequence_parallel_region,
26
+ reduce_from_tensor_model_parallel_region,
27
+ scatter_to_tensor_model_parallel_region,
28
+ reduce_scatter_to_sequence_parallel_region,
29
+)
30
+
31
+from .random import get_cuda_rng_tracker
32
+from .utils import (
33
+ divide,
34
+ split_tensor_along_last_dim,
35
+ VocabUtility,
36
+)
37
+
38
+_grad_accum_fusion_available = True
39
+try:
40
+ import fused_weight_gradient_mlp_cuda
41
+except ImportError:
42
+ _grad_accum_fusion_available = False
43
+
44
+_MODEL_PARALLEL_ATTRIBUTE_DEFAULTS = {'tensor_model_parallel': False,
45
+ 'partition_dim': -1,
46
+ 'partition_stride': 1}
47
+
48
+def param_is_not_tensor_parallel_duplicate(param):
49
+ return (hasattr(param, 'tensor_model_parallel') and
50
+ param.tensor_model_parallel) or (
51
+ get_tensor_model_parallel_rank() == 0)
52
+
53
+
54
+def set_tensor_model_parallel_attributes(tensor, is_parallel, dim, stride):
55
+ # Make sure the attributes are not set.
56
+ for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS:
57
+ assert not hasattr(tensor, attribute)
58
+ # Set the attributes.
59
+ setattr(tensor, 'tensor_model_parallel', is_parallel)
60
+ setattr(tensor, 'partition_dim', dim)
61
+ setattr(tensor, 'partition_stride', stride)
62
+
63
+
64
+def set_defaults_if_not_set_tensor_model_parallel_attributes(tensor):
65
+ def maybe_set(attribute, value):
66
+ if not hasattr(tensor, attribute):
67
+ setattr(tensor, attribute, value)
68
+ for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS:
69
+ maybe_set(attribute, _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS[attribute])
70
+
71
+
72
+def copy_tensor_model_parallel_attributes(destination_tensor, source_tensor):
73
+ def maybe_copy(attribute):
74
+ if hasattr(source_tensor, attribute):
75
+ setattr(destination_tensor, attribute,
76
+ getattr(source_tensor, attribute))
77
+ for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS:
78
+ maybe_copy(attribute)
79
+
80
+
81
+def _initialize_affine_weight_gpu(weight, init_method,
82
+ partition_dim, stride=1):
83
+ """Initialize affine weight for model parallel on GPU."""
84
+
85
+ set_tensor_model_parallel_attributes(tensor=weight,
86
+ is_parallel=True,
87
+ dim=partition_dim,
88
+ stride=stride)
89
+
90
+ with get_cuda_rng_tracker().fork():
91
+ init_method(weight)
92
+
93
+
94
+def _initialize_affine_weight_cpu(weight, output_size, input_size,
95
+ per_partition_size, partition_dim,
96
+ init_method, stride=1,
97
+ return_master_weight=False,
98
+ *, params_dtype=None):
99
+ """Initialize affine weight for model parallel.
100
+
101
+ Build the master weight on all processes and scatter
102
+ the relevant chunk."""
103
+
104
+ set_tensor_model_parallel_attributes(tensor=weight,
105
+ is_parallel=True,
106
+ dim=partition_dim,
107
+ stride=stride)
108
+
109
+ if params_dtype is None:
110
+ params_dtype = torch.get_default_dtype()
111
+
112
+ # Initialize master weight
113
+ master_weight = torch.empty(output_size, input_size,
114
+ dtype=torch.float,
115
+ requires_grad=False)
116
+ init_method(master_weight)
117
+ master_weight = master_weight.to(dtype=params_dtype)
118
+
119
+ # Split and copy
120
+ per_partition_per_stride_size = divide(per_partition_size, stride)
121
+ weight_list = torch.split(master_weight, per_partition_per_stride_size,
122
+ dim=partition_dim)
123
+ rank = get_tensor_model_parallel_rank()
124
+ world_size = get_tensor_model_parallel_world_size()
125
+ my_weight_list = weight_list[rank::world_size]
126
+
127
+ with torch.no_grad():
128
+ torch.cat(my_weight_list, dim=partition_dim, out=weight)
129
+ if return_master_weight:
130
+ return master_weight
131
+ return None
132
+
133
+
134
+class VocabParallelEmbedding(torch.nn.Module):
135
+ """Embedding parallelized in the vocabulary dimension.
136
+
137
+ This is mainly adapted from torch.nn.Embedding and all the default
138
+ values are kept.
139
+ Arguments:
140
+ num_embeddings: vocabulary size.
141
+ embedding_dim: size of hidden state.
142
+
143
+ Keyword Arguments:
144
+ init_method: method to initialize weights.
145
+ params_dtype
146
+ use_cpu_initialization
147
+ perform_initialization
148
+ """
149
+
150
+ def __init__(self, num_embeddings: int, embedding_dim: int, *,
151
+ init_method=init.xavier_normal_,
152
+ params_dtype: torch.dtype=None,
153
+ use_cpu_initialization: bool=False,
154
+ perform_initialization: bool=True):
155
+ super(VocabParallelEmbedding, self).__init__()
156
+ # Keep the input dimensions.
157
+ self.num_embeddings = num_embeddings
158
+ self.embedding_dim = embedding_dim
159
+ if params_dtype is None:
160
+ params_dtype = torch.get_default_dtype()
161
+
162
+ # Set the defaults for compatibility.
163
+ self.padding_idx = None
164
+ self.max_norm = None
165
+ self.norm_type = 2.
166
+ self.scale_grad_by_freq = False
167
+ self.sparse = False
168
+ self._weight = None
169
+ self.tensor_model_parallel_size = get_tensor_model_parallel_world_size()
170
+ # Divide the weight matrix along the vocaburaly dimension.
171
+ self.vocab_start_index, self.vocab_end_index = \
172
+ VocabUtility.vocab_range_from_global_vocab_size(
173
+ self.num_embeddings, get_tensor_model_parallel_rank(),
174
+ self.tensor_model_parallel_size)
175
+ self.num_embeddings_per_partition = self.vocab_end_index - \
176
+ self.vocab_start_index
177
+
178
+ # Allocate weights and initialize.
179
+ if use_cpu_initialization:
180
+ self.weight = Parameter(torch.empty(
181
+ self.num_embeddings_per_partition, self.embedding_dim,
182
+ dtype=params_dtype))
183
+ if perform_initialization:
184
+ _initialize_affine_weight_cpu(
185
+ self.weight, self.num_embeddings, self.embedding_dim,
186
+ self.num_embeddings_per_partition, 0, init_method,
187
+ params_dtype=params_dtype)
188
+ else:
189
+ self.weight = Parameter(torch.empty(
190
+ self.num_embeddings_per_partition, self.embedding_dim,
191
+ device=torch.cuda.current_device(), dtype=params_dtype))
192
+ if perform_initialization:
193
+ _initialize_affine_weight_gpu(self.weight, init_method,
194
+ partition_dim=0, stride=1)
195
+
196
+ def forward(self, input_):
197
+ if self.tensor_model_parallel_size > 1:
198
+ # Build the mask.
199
+ input_mask = (input_ < self.vocab_start_index) | \
200
+ (input_ >= self.vocab_end_index)
201
+ # Mask the input.
202
+ masked_input = input_.clone() - self.vocab_start_index
203
+ masked_input[input_mask] = 0
204
+ else:
205
+ masked_input = input_
206
+ # Get the embeddings.
207
+ output_parallel = F.embedding(masked_input, self.weight,
208
+ self.padding_idx, self.max_norm,
209
+ self.norm_type, self.scale_grad_by_freq,
210
+ self.sparse)
211
+ # Mask the output embedding.
212
+ if self.tensor_model_parallel_size > 1:
213
+ output_parallel[input_mask, :] = 0.0
214
+ # Reduce across all the model parallel GPUs.
215
+ output = reduce_from_tensor_model_parallel_region(output_parallel)
216
+ return output
217
+
218
+
219
+class LinearWithGradAccumulationAndAsyncCommunication(torch.autograd.Function):
220
+ """See linear_with_grad_accumulation_and_async_allreduce"""
221
+
222
+ @staticmethod
223
+ def forward(ctx, input, weight, bias, gradient_accumulation_fusion,
224
+ async_grad_allreduce, sequence_parallel):
225
+ ctx.save_for_backward(input, weight)
226
+ ctx.use_bias = bias is not None
227
+ ctx.gradient_accumulation_fusion = gradient_accumulation_fusion
228
+ ctx.async_grad_allreduce = async_grad_allreduce
229
+ ctx.sequence_parallel = sequence_parallel
230
+
231
+ if sequence_parallel:
232
+ world_size = get_tensor_model_parallel_world_size()
233
+ dim_size = list(input.size())
234
+ dim_size[0] = dim_size[0] * world_size
235
+
236
+ all_gather_buffer = \
237
+ get_global_memory_buffer().get_tensor(dim_size, input.dtype, "mpu")
238
+ torch.distributed._all_gather_base(
239
+ all_gather_buffer,
240
+ input,
241
+ group=get_tensor_model_parallel_group())
242
+ total_input = all_gather_buffer
243
+ else:
244
+ total_input = input
245
+
246
+ output = torch.matmul(total_input, weight.t())
247
+ if bias is not None:
248
+ output = output + bias
249
+ return output
250
+
251
+ @staticmethod
252
+ def backward(ctx, grad_output):
253
+ input, weight = ctx.saved_tensors
254
+ use_bias = ctx.use_bias
255
+
256
+ if ctx.sequence_parallel:
257
+ world_size = get_tensor_model_parallel_world_size()
258
+ dim_size = list(input.size())
259
+ dim_size[0] = dim_size[0] * world_size
260
+
261
+ all_gather_buffer = \
262
+ get_global_memory_buffer().get_tensor(dim_size, input.dtype, "mpu")
263
+ handle = torch.distributed._all_gather_base(
264
+ all_gather_buffer,
265
+ input,
266
+ group=get_tensor_model_parallel_group(), async_op=True)
267
+
268
+ # Here we rely on CUDA_DEVICE_MAX_CONNECTIONS=1 to ensure that the
269
+ # gather is scheduled before the input gradient computation
270
+ total_input = all_gather_buffer
271
+ else:
272
+ total_input = input
273
+ grad_input = grad_output.matmul(weight)
274
+
275
+ if ctx.sequence_parallel:
276
+ handle.wait()
277
+
278
+ # Convert the tensor shapes to 2D for execution compatibility
279
+ grad_output = grad_output.view(grad_output.shape[0] * grad_output.shape[1],
280
+ grad_output.shape[2])
281
+ total_input = total_input.view(total_input.shape[0] * total_input.shape[1],
282
+ total_input.shape[2])
283
+
284
+ if ctx.async_grad_allreduce:
285
+ # Asynchronous all-reduce
286
+ handle = torch.distributed.all_reduce(
287
+ grad_input, group=get_tensor_model_parallel_group(), async_op=True)
288
+ # Here we rely on CUDA_DEVICE_MAX_CONNECTIONS=1 to ensure that the
289
+ # all-reduce is scheduled before the weight gradient computation
290
+
291
+ if ctx.sequence_parallel:
292
+ assert not ctx.async_grad_allreduce
293
+ dim_size = list(input.size())
294
+ sub_grad_input = torch.empty(dim_size, dtype=input.dtype,
295
+ device=torch.cuda.current_device(),
296
+ requires_grad=False)
297
+ # reduce_scatter
298
+ handle = torch.distributed._reduce_scatter_base(sub_grad_input, grad_input,
299
+ group=get_tensor_model_parallel_group(),
300
+ async_op=True)
301
+ # Here we rely on CUDA_DEVICE_MAX_CONNECTIONS=1 to ensure that the
302
+ # reduce scatter is scheduled before the weight gradient computation
303
+
304
+
305
+ if ctx.gradient_accumulation_fusion:
306
+ if weight.main_grad.dtype == torch.float32:
307
+ fused_weight_gradient_mlp_cuda.wgrad_gemm_accum_fp32(total_input, grad_output, weight.main_grad)
308
+ elif weight.main_grad.dtype == torch.float16:
309
+ fused_weight_gradient_mlp_cuda.wgrad_gemm_accum_fp16(total_input, grad_output, weight.main_grad)
310
+ else:
311
+ raise RuntimeError("Unsupported gradient type for gradient accumulation fusion")
312
+ grad_weight = None
313
+ else:
314
+ grad_weight = grad_output.t().matmul(total_input)
315
+ grad_bias = grad_output.sum(dim=0) if use_bias else None
316
+
317
+ if ctx.sequence_parallel:
318
+ handle.wait()
319
+ return sub_grad_input, grad_weight, grad_bias, None, None, None
320
+
321
+ if ctx.async_grad_allreduce:
322
+ handle.wait()
323
+
324
+ return grad_input, grad_weight, grad_bias, None, None, None
325
+
326
+def linear_with_grad_accumulation_and_async_allreduce(
327
+ input: torch.Tensor,
328
+ weight: torch.Tensor,
329
+ bias: Optional[torch.Tensor],
330
+ gradient_accumulation_fusion: bool,
331
+ async_grad_allreduce: bool,
332
+ sequence_parallel_enabled: bool,
333
+) -> torch.Tensor:
334
+ """Linear layer execution with asynchronous communication and
335
+ gradient accumulation fusion in backprop.
336
+
337
+ This has the option to accumulate the result of backprop
338
+ calculation into an existing gradient buffer, preventing the need
339
+ to do an additional addition kernel after the gradient
340
+ calculation.
341
+
342
+ Additionally, the tensor parallel all reduce of the input
343
+ gradients can be done asynchronously with the calculation of
344
+ the weight gradients.
345
+
346
+ In the case of sequence parallelism, the reduce scatter of the
347
+ input gradients is done asynchronously with the calcluation of the
348
+ weight gradients.
349
+
350
+ Use of this module requires that the environment variable
351
+ CUDA_DEVICE_MAX_CONNECTIONS=1. There are a few collective
352
+ operations, noted in the code, that should be scheduled before
353
+ compute kernels to overlap the communication with the computation,
354
+ which is necessary for a speedup but not for correctness so that
355
+ ordering isn't imposed by the scheduler. Setting
356
+ CUDA_DEVICE_MAX_CONNECTIONS=1 forces the kernels to be scheduled
357
+ in the order they are called.
358
+
359
+ Arguments:
360
+
361
+ input (torch.Tensor required): input like torch.nn.functional.linear
362
+
363
+ weight (torch.Tensor required): weight like torch.nn.functional.linear
364
+
365
+ bias (torch.Tensor optional): bias like torch.nn.functional.linear
366
+
367
+ gradient_accumulation_fusion (bool required): Perform the gradient
368
+ accumulation fusion, requires the custom CUDA extension
369
+ fused_weight_gradient_mlp_cuda module. To use
370
+ gradient_accumulation_fusion you must install APEX with
371
+ --cpp_ext and --cuda_ext. For example: "pip install
372
+ --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext .\"
373
+ " Note that the extension requires CUDA>=11. Otherwise, you
374
+ must turn off gradient accumulation fusion."
375
+
376
+ async_grad_allreduce (bool required): Do the allreduce of input
377
+ gradients asyncronously with the computation of weight
378
+ gradients. If sequence_parallel_enabled is True, this must be
379
+ False, as no all reduce is performed.
380
+
381
+ sequence_parallel_enabled (bool required): Indicates that sequence
382
+ parallelism is used and thus in the forward pass the input is
383
+ all gathered, and the backward pass the input gradients are
384
+ reduce scattered.
385
+ """
386
+ args = [
387
+ input,
388
+ weight,
389
+ bias,
390
+ gradient_accumulation_fusion,
391
+ async_grad_allreduce,
392
+ sequence_parallel_enabled,
393
+ ]
394
+
395
+ if not linear_with_grad_accumulation_and_async_allreduce.warned:
396
+ if os.environ.get('CUDA_DEVICE_MAX_CONNECTIONS') != "1":
397
+ if sequence_parallel_enabled:
398
+ warnings.warn(
399
+ "When using sequence parallelism it is recommended to set the "
400
+ "environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 for "
401
+ "maximum speedup")
402
+ linear_with_grad_accumulation_and_async_allreduce.warned = True
403
+
404
+ if async_grad_allreduce:
405
+ warnings.warn(
406
+ "When using async grad allreduce it is recommended to set the "
407
+ "environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 for "
408
+ "maximum speedup")
409
+ linear_with_grad_accumulation_and_async_allreduce.warned = True
410
+
411
+ with torch.cuda.amp.autocast(enabled=False):
412
+ return LinearWithGradAccumulationAndAsyncCommunication.apply(*args)
413
+linear_with_grad_accumulation_and_async_allreduce.warned = False
414
+
415
+class ColumnParallelLinear(torch.nn.Module):
416
+ """Linear layer with column parallelism.
417
+
418
+ The linear layer is defined as Y = XA + b. A is parallelized along
419
+ its second dimension as A = [A_1, ..., A_p].
420
+
421
+ Arguments:
422
+ input_size: first dimension of matrix A.
423
+ output_size: second dimension of matrix A.
424
+
425
+ Keyword Arguments
426
+ bias: If true, add bias
427
+ gather_output: If true, call all-gather on output and make Y available
428
+ to all GPUs, otherwise, every GPU will have its output
429
+ which is Y_i = XA_i
430
+ init_method: method to initialize weights. Note that bias is always set
431
+ to zero.
432
+ stride: For the strided linear layers.
433
+ keep_master_weight_for_test: This was added for testing and should be
434
+ set to False. It returns the master weights
435
+ used for initialization.
436
+ skip_bias_add: This was added to enable performance optimations where bias
437
+ can be fused with other elementwise operations. we skip
438
+ adding bias but instead return it.
439
+ async_tensor_model_parallel_allreduce:
440
+ params_dtype:
441
+ use_cpu_initialization:
442
+ gradient_accumulation_fusion:
443
+ sequence_parallel_enabled:
444
+ """
445
+
446
+ def __init__(self, input_size, output_size, *,
447
+ bias=True, gather_output=True,
448
+ init_method=init.xavier_normal_, stride=1,
449
+ keep_master_weight_for_test=False,
450
+ skip_bias_add=False,
451
+ async_tensor_model_parallel_allreduce=True,
452
+ params_dtype=None,
453
+ use_cpu_initialization=False,
454
+ perform_initialization=True,
455
+ gradient_accumulation_fusion=False,
456
+ sequence_parallel_enabled: bool = False,
457
+ ):
458
+ super(ColumnParallelLinear, self).__init__()
459
+
460
+ # Keep input parameters
461
+ self.input_size = input_size
462
+ self.output_size = output_size
463
+ self.gather_output = gather_output
464
+ # Divide the weight matrix along the last dimension.
465
+ world_size = get_tensor_model_parallel_world_size()
466
+ self.output_size_per_partition = divide(output_size, world_size)
467
+ self.skip_bias_add = skip_bias_add
468
+
469
+ if params_dtype is None:
470
+ params_dtype = torch.get_default_dtype()
471
+
472
+ # Parameters.
473
+ # Note: torch.nn.functional.linear performs XA^T + b and as a result
474
+ # we allocate the transpose.
475
+ # Initialize weight.
476
+ if use_cpu_initialization:
477
+ self.weight = Parameter(torch.empty(self.output_size_per_partition,
478
+ self.input_size,
479
+ dtype=params_dtype))
480
+ if perform_initialization:
481
+ self.master_weight = _initialize_affine_weight_cpu(
482
+ self.weight, self.output_size, self.input_size,
483
+ self.output_size_per_partition, 0, init_method,
484
+ stride=stride, return_master_weight=keep_master_weight_for_test)
485
+ else:
486
+ self.weight = Parameter(torch.empty(
487
+ self.output_size_per_partition, self.input_size,
488
+ device=torch.cuda.current_device(), dtype=params_dtype))
489
+ if perform_initialization:
490
+ _initialize_affine_weight_gpu(self.weight, init_method,
491
+ partition_dim=0, stride=stride)
492
+
493
+ if bias:
494
+ if use_cpu_initialization:
495
+ self.bias = Parameter(torch.empty(
496
+ self.output_size_per_partition, dtype=params_dtype))
497
+ else:
498
+ self.bias = Parameter(torch.empty(
499
+ self.output_size_per_partition,
500
+ device=torch.cuda.current_device(),
501
+ dtype=params_dtype))
502
+ set_tensor_model_parallel_attributes(self.bias, True, 0, stride)
503
+ # Always initialize bias to zero.
504
+ with torch.no_grad():
505
+ self.bias.zero_()
506
+ else:
507
+ self.register_parameter('bias', None)
508
+
509
+ self.async_tensor_model_parallel_allreduce = (
510
+ async_tensor_model_parallel_allreduce and
511
+ world_size > 1)
512
+ if sequence_parallel_enabled:
513
+ if world_size <= 1:
514
+ warnings.warn(
515
+ f"`sequence_parallel_enabled` is set to `True`, but tensor model parallel size is {world_size}. "
516
+ f"Disabling sequence parallel."
517
+ )
518
+ sequence_parallel_enabled = False
519
+ self.sequence_parallel_enabled = sequence_parallel_enabled
520
+
521
+ if gradient_accumulation_fusion:
522
+ if not _grad_accum_fusion_available:
523
+ raise RuntimeError(
524
+ "ColumnParallelLinear was called with gradient_accumulation_fusion set "
525
+ "to True but the custom CUDA extension fused_weight_gradient_mlp_cuda "
526
+ "module is not found. To use gradient_accumulation_fusion you must "
527
+ "install APEX with --cpp_ext and --cuda_ext. For example: "
528
+ "pip install --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext .\" "
529
+ "Note that the extension requires CUDA>=11. Otherwise, you must turn off "
530
+ "gradient accumulation fusion."
531
+ )
532
+ self.gradient_accumulation_fusion = gradient_accumulation_fusion
533
+
534
+ if self.async_tensor_model_parallel_allreduce and self.sequence_parallel_enabled:
535
+ raise RuntimeError(
536
+ "`async_tensor_model_parallel_allreduce` and `sequence_parallel_enabled` "
537
+ "cannot be enabled at the same time."
538
+ )
539
+
540
+
541
+ def forward(self, input_):
542
+ """Forward of ColumnParallelLinear
543
+
544
+ Args:
545
+ input_: 3D tensor whose order of dimension is [sequence, batch, hidden]
546
+
547
+ Returns:
548
+ - output
549
+ - bias
550
+ """
551
+ bias = self.bias if not self.skip_bias_add else None
552
+
553
+ if self.async_tensor_model_parallel_allreduce or \
554
+ self.sequence_parallel_enabled:
555
+ input_parallel = input_
556
+ else:
557
+ input_parallel = copy_to_tensor_model_parallel_region(input_)
558
+ # Matrix multiply.
559
+ output_parallel = linear_with_grad_accumulation_and_async_allreduce(
560
+ input=input_parallel,
561
+ weight=self.weight,
562
+ bias=bias,
563
+ gradient_accumulation_fusion=self.gradient_accumulation_fusion,
564
+ async_grad_allreduce=self.async_tensor_model_parallel_allreduce,
565
+ sequence_parallel_enabled=self.sequence_parallel_enabled,
566
+ )
567
+ if self.gather_output:
568
+ # All-gather across the partitions.
569
+ assert not self.sequence_parallel_enabled
570
+ output = gather_from_tensor_model_parallel_region(output_parallel)
571
+ else:
572
+ output = output_parallel
573
+ output_bias = self.bias if self.skip_bias_add else None
574
+ return output, output_bias
575
+
576
+
577
+class RowParallelLinear(torch.nn.Module):
578
+ """Linear layer with row parallelism.
579
+
580
+ The linear layer is defined as Y = XA + b. A is parallelized along
581
+ its first dimension and X along its second dimension as:
582
+ - -
583
+ | A_1 |
584
+ | . |
585
+ A = | . | X = [X_1, ..., X_p]
586
+ | . |
587
+ | A_p |
588
+ - -
589
+ Arguments:
590
+ input_size: first dimension of matrix A.
591
+ output_size: second dimension of matrix A.
592
+
593
+ Keyword Arguments:
594
+ bias: If true, add bias. Note that bias is not parallelized.
595
+ input_is_parallel: If true, we assume that the input is already
596
+ split across the GPUs and we do not split
597
+ again.
598
+ init_method: method to initialize weights. Note that bias is always set
599
+ to zero.
600
+ stride: For the strided linear layers.
601
+ keep_master_weight_for_test: This was added for testing and should be
602
+ set to False. It returns the master weights
603
+ used for initialization.
604
+ skip_bias_add: This was added to enable performance optimization where bias
605
+ can be fused with other elementwise operations. We skip
606
+ adding bias but instead return it.
607
+ params_dtype:
608
+ use_cpu_initialization:
609
+ perform_initialization:
610
+ gradient_accumulation_fusion:
611
+ sequence_parallel_enabled:
612
+ """
613
+
614
+ def __init__(self, input_size, output_size, *,
615
+ bias=True, input_is_parallel=False,
616
+ init_method=init.xavier_normal_, stride=1,
617
+ keep_master_weight_for_test=False,
618
+ skip_bias_add=False,
619
+ params_dtype=None,
620
+ use_cpu_initialization=False,
621
+ perform_initialization=True,
622
+ gradient_accumulation_fusion=False,
623
+ sequence_parallel_enabled: bool = False,
624
+ ):
625
+ super(RowParallelLinear, self).__init__()
626
+
627
+ # Keep input parameters
628
+ self.input_size = input_size
629
+ self.output_size = output_size
630
+ self.input_is_parallel = input_is_parallel
631
+ if params_dtype is None:
632
+ params_dtype = torch.get_default_dtype()
633
+
634
+ # Divide the weight matrix along the last dimension.
635
+ world_size = get_tensor_model_parallel_world_size()
636
+ self.input_size_per_partition = divide(input_size, world_size)
637
+ self.skip_bias_add = skip_bias_add
638
+ self.gradient_accumulation_fusion = gradient_accumulation_fusion
639
+ self.sequence_parallel_enabled = sequence_parallel_enabled
640
+ if self.sequence_parallel_enabled and not self.input_is_parallel:
641
+ raise RuntimeError("To enable `sequence_parallel_enabled`, `input_is_parallel` must be `True`")
642
+
643
+ # Parameters.
644
+ # Note: torch.nn.functional.linear performs XA^T + b and as a result
645
+ # we allocate the transpose.
646
+ # Initialize weight.
647
+ if use_cpu_initialization:
648
+ self.weight = Parameter(torch.empty(self.output_size,
649
+ self.input_size_per_partition,
650
+ dtype=params_dtype))
651
+ if perform_initialization:
652
+ self.master_weight = _initialize_affine_weight_cpu(
653
+ self.weight, self.output_size, self.input_size,
654
+ self.input_size_per_partition, 1, init_method,
655
+ stride=stride, return_master_weight=keep_master_weight_for_test,
656
+ params_dtype=params_dtype)
657
+ else:
658
+ self.weight = Parameter(torch.empty(
659
+ self.output_size, self.input_size_per_partition,
660
+ device=torch.cuda.current_device(), dtype=params_dtype))
661
+ if perform_initialization:
662
+ _initialize_affine_weight_gpu(self.weight, init_method,
663
+ partition_dim=1, stride=stride)
664
+ if bias:
665
+ if use_cpu_initialization:
666
+ self.bias = Parameter(torch.empty(self.output_size,
667
+ dtype=params_dtype))
668
+ else:
669
+ self.bias = Parameter(torch.empty(
670
+ self.output_size, device=torch.cuda.current_device(),
671
+ dtype=params_dtype))
672
+ setattr(self.bias, 'sequence_parallel', sequence_parallel_enabled)
673
+
674
+ # Always initialize bias to zero.
675
+ with torch.no_grad():
676
+ self.bias.zero_()
677
+ else:
678
+ self.register_parameter('bias', None)
679
+
680
+
681
+
682
+ def forward(self, input_):
683
+ """Forward of RowParallelLinear
684
+
685
+ Args:
686
+ input_: 3D tensor whose order of dimension is [sequence, batch, hidden]
687
+
688
+ Returns:
689
+ - output
690
+ - bias
691
+ """
692
+ # Set up backprop all-reduce.
693
+ if self.input_is_parallel:
694
+ input_parallel = input_
695
+ else:
696
+ assert not self.sequence_parallel_enabled
697
+ input_parallel = scatter_to_tensor_model_parallel_region(input_)
698
+ # Matrix multiply.
699
+ output_parallel = linear_with_grad_accumulation_and_async_allreduce(
700
+ input=input_parallel,
701
+ weight=self.weight,
702
+ bias=None,
703
+ gradient_accumulation_fusion=self.gradient_accumulation_fusion,
704
+ async_grad_allreduce=False,
705
+ sequence_parallel_enabled=False,
706
+ )
707
+
708
+ # All-reduce across all the partitions.
709
+ if self.sequence_parallel_enabled:
710
+ output_ = reduce_scatter_to_sequence_parallel_region(output_parallel)
711
+ else:
712
+ output_ = reduce_from_tensor_model_parallel_region(output_parallel)
713
+ if not self.skip_bias_add:
714
+ output = output_ + self.bias if self.bias is not None else output_
715
+ output_bias = None
716
+ else:
717
+ output = output_
718
+ output_bias = self.bias
719
+ return output, output_bias
cacheflow/parallel_utils/tensor_parallel/mappings.py
new
+279
@@ -0,0 +1,279 @@
1
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2
+
3
+import torch
4
+
5
+from cacheflow.parallel_utils.parallel_state import (
6
+ get_tensor_model_parallel_rank,
7
+ get_tensor_model_parallel_world_size,
8
+ get_tensor_model_parallel_group,
9
+)
10
+from .utils import split_tensor_along_last_dim
11
+
12
+
13
+def _reduce(input_):
14
+ """All-reduce the input tensor across model parallel group."""
15
+
16
+ # Bypass the function if we are using only 1 GPU.
17
+ if get_tensor_model_parallel_world_size()==1:
18
+ return input_
19
+
20
+ # All-reduce.
21
+ torch.distributed.all_reduce(input_, group=get_tensor_model_parallel_group())
22
+
23
+ return input_
24
+
25
+
26
+def _split_along_last_dim(input_):
27
+ """Split the tensor along its last dimension and keep the
28
+ corresponding slice."""
29
+
30
+ world_size = get_tensor_model_parallel_world_size()
31
+ # Bypass the function if we are using only 1 GPU.
32
+ if world_size == 1:
33
+ return input_
34
+
35
+ # Split along last dimension.
36
+ input_list = split_tensor_along_last_dim(input_, world_size)
37
+
38
+ # Note: torch.split does not create contiguous tensors by default.
39
+ rank = get_tensor_model_parallel_rank()
40
+ output = input_list[rank].contiguous()
41
+
42
+ return output
43
+
44
+
45
+def _split_along_first_dim(input_):
46
+ """Split the tensor along its first dimension and keep the
47
+ corresponding slice."""
48
+
49
+ world_size = get_tensor_model_parallel_world_size()
50
+ # Bypass the function if we are using only 1 GPU.
51
+ if world_size == 1:
52
+ return input_
53
+
54
+ # Split along first dimension.
55
+ dim_size = input_.size()[0]
56
+ assert dim_size % world_size == 0, \
57
+ "First dimension of the tensor should be divisible by tensor parallel size"
58
+ local_dim_size = dim_size // world_size
59
+ rank = get_tensor_model_parallel_rank()
60
+ dim_offset = rank * local_dim_size
61
+
62
+ output = input_[dim_offset:dim_offset+local_dim_size].contiguous()
63
+
64
+ return output
65
+
66
+
67
+def _gather_along_last_dim(input_):
68
+ """Gather tensors and concatinate along the last dimension."""
69
+
70
+ world_size = get_tensor_model_parallel_world_size()
71
+ # Bypass the function if we are using only 1 GPU.
72
+ if world_size == 1:
73
+ return input_
74
+
75
+ # Size and dimension.
76
+ last_dim = input_.dim() - 1
77
+ rank = get_tensor_model_parallel_rank()
78
+
79
+ tensor_list = [torch.empty_like(input_) for _ in range(world_size)]
80
+ tensor_list[rank] = input_
81
+ torch.distributed.all_gather(tensor_list, input_, group=get_tensor_model_parallel_group())
82
+
83
+ # Note: torch.cat already creates a contiguous tensor.
84
+ output = torch.cat(tensor_list, dim=last_dim).contiguous()
85
+
86
+ return output
87
+
88
+
89
+def _gather_along_first_dim(input_):
90
+ """Gather tensors and concatinate along the first dimension."""
91
+
92
+ world_size = get_tensor_model_parallel_world_size()
93
+ # Bypass the function if we are using only 1 GPU.
94
+ if world_size == 1:
95
+ return input_
96
+
97
+ dim_size = list(input_.size())
98
+ dim_size[0] = dim_size[0] * world_size
99
+
100
+ output = torch.empty(dim_size, dtype=input_.dtype,
101
+ device=torch.cuda.current_device())
102
+ torch.distributed._all_gather_base(output, input_.contiguous(),
103
+ group=get_tensor_model_parallel_group())
104
+
105
+ return output
106
+
107
+def _reduce_scatter_along_first_dim(input_):
108
+ """Reduce-scatter the input tensor across model parallel group."""
109
+ world_size = get_tensor_model_parallel_world_size()
110
+ # Bypass the function if we are using only 1 GPU.
111
+ if world_size == 1:
112
+ return input_
113
+
114
+ dim_size = list(input_.size())
115
+ assert dim_size[0] % world_size == 0, \
116
+ "First dimension of the tensor should be divisible by tensor parallel size"
117
+
118
+ dim_size[0] = dim_size[0] // world_size
119
+
120
+ output = torch.empty(dim_size, dtype=input_.dtype,
121
+ device=torch.cuda.current_device())
122
+ torch.distributed._reduce_scatter_base(output, input_.contiguous(),
123
+ group=get_tensor_model_parallel_group())
124
+ return output
125
+
126
+
127
+class _CopyToModelParallelRegion(torch.autograd.Function):
128
+ """Pass the input to the model parallel region."""
129
+
130
+ @staticmethod
131
+ def symbolic(graph, input_):
132
+ return input_
133
+
134
+ @staticmethod
135
+ def forward(ctx, input_):
136
+ return input_
137
+
138
+ @staticmethod
139
+ def backward(ctx, grad_output):
140
+ return _reduce(grad_output)
141
+
142
+
143
+class _ReduceFromModelParallelRegion(torch.autograd.Function):
144
+ """All-reduce the input from the model parallel region."""
145
+
146
+ @staticmethod
147
+ def symbolic(graph, input_):
148
+ return _reduce(input_)
149
+
150
+ @staticmethod
151
+ def forward(ctx, input_):
152
+ return _reduce(input_)
153
+
154
+ @staticmethod
155
+ def backward(ctx, grad_output):
156
+ return grad_output
157
+
158
+
159
+class _ScatterToModelParallelRegion(torch.autograd.Function):
160
+ """Split the input and keep only the corresponding chuck to the rank."""
161
+
162
+ @staticmethod
163
+ def symbolic(graph, input_):
164
+ return _split_along_last_dim(input_)
165
+
166
+ @staticmethod
167
+ def forward(ctx, input_):
168
+ return _split_along_last_dim(input_)
169
+
170
+ @staticmethod
171
+ def backward(ctx, grad_output):
172
+ return _gather_along_last_dim(grad_output)
173
+
174
+
175
+class _GatherFromModelParallelRegion(torch.autograd.Function):
176
+ """Gather the input from model parallel region and concatinate."""
177
+
178
+ @staticmethod
179
+ def symbolic(graph, input_):
180
+ return _gather_along_last_dim(input_)
181
+
182
+ @staticmethod
183
+ def forward(ctx, input_):
184
+ return _gather_along_last_dim(input_)
185
+
186
+ @staticmethod
187
+ def backward(ctx, grad_output):
188
+ return _split_along_last_dim(grad_output)
189
+
190
+
191
+class _ScatterToSequenceParallelRegion(torch.autograd.Function):
192
+ """Split the input and keep only the corresponding chuck to the rank."""
193
+
194
+ @staticmethod
195
+ def symbolic(graph, input_):
196
+ return _split_along_first_dim(input_)
197
+
198
+ @staticmethod
199
+ def forward(ctx, input_):
200
+ return _split_along_first_dim(input_)
201
+
202
+ @staticmethod
203
+ def backward(ctx, grad_output):
204
+ return _gather_along_first_dim(grad_output)
205
+
206
+
207
+class _GatherFromSequenceParallelRegion(torch.autograd.Function):
208
+ """Gather the input from sequence parallel region and concatinate."""
209
+
210
+ @staticmethod
211
+ def symbolic(graph, input_, tensor_parallel_output_grad=True):
212
+ return _gather_along_first_dim(input_)
213
+
214
+ @staticmethod
215
+ def forward(ctx, input_, tensor_parallel_output_grad=True):
216
+ ctx.tensor_parallel_output_grad = tensor_parallel_output_grad
217
+ return _gather_along_first_dim(input_)
218
+
219
+ @staticmethod
220
+ def backward(ctx, grad_output):
221
+ tensor_parallel_output_grad = ctx.tensor_parallel_output_grad
222
+
223
+ # If the computation graph after the gather operation is
224
+ # in the tensor parallel mode, output gradients need to reduce
225
+ # scattered and whereas if the computation is duplicated,
226
+ # output gradients need to be scattered.
227
+ if tensor_parallel_output_grad:
228
+ return _reduce_scatter_along_first_dim(grad_output), None
229
+ else:
230
+ return _split_along_first_dim(grad_output), None
231
+
232
+
233
+class _ReduceScatterToSequenceParallelRegion(torch.autograd.Function):
234
+ """Reduce scatter the input from the model parallel region."""
235
+
236
+ @staticmethod
237
+ def symbolic(graph, input_):
238
+ return _reduce_scatter_along_first_dim(input_)
239
+
240
+ @staticmethod
241
+ def forward(ctx, input_):
242
+ return _reduce_scatter_along_first_dim(input_)
243
+
244
+ @staticmethod
245
+ def backward(ctx, grad_output):
246
+ return _gather_along_first_dim(grad_output)
247
+
248
+
249
+# -----------------
250
+# Helper functions.
251
+# -----------------
252
+
253
+def copy_to_tensor_model_parallel_region(input_):
254
+ return _CopyToModelParallelRegion.apply(input_)
255
+
256
+
257
+def reduce_from_tensor_model_parallel_region(input_):
258
+ return _ReduceFromModelParallelRegion.apply(input_)
259
+
260
+
261
+def scatter_to_tensor_model_parallel_region(input_):
262
+ return _ScatterToModelParallelRegion.apply(input_)
263
+
264
+
265
+def gather_from_tensor_model_parallel_region(input_):
266
+ return _GatherFromModelParallelRegion.apply(input_)
267
+
268
+
269
+def scatter_to_sequence_parallel_region(input_):
270
+ return _ScatterToSequenceParallelRegion.apply(input_)
271
+
272
+
273
+def gather_from_sequence_parallel_region(input_, tensor_parallel_output_grad=True):
274
+ return _GatherFromSequenceParallelRegion.apply(input_, tensor_parallel_output_grad)
275
+
276
+
277
+def reduce_scatter_to_sequence_parallel_region(input_):
278
+ return _ReduceScatterToSequenceParallelRegion.apply(input_)
279
+
cacheflow/parallel_utils/tensor_parallel/random.py
new
+253
@@ -0,0 +1,253 @@
1
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2
+
3
+# Parts of the code here are adapted from PyTorch
4
+# repo: https://github.com/pytorch/pytorch
5
+
6
+import contextlib
7
+
8
+import torch
9
+from torch import _C
10
+from torch.cuda import _lazy_call, device as device_ctx_manager
11
+from torch.utils.checkpoint import detach_variable
12
+
13
+from cacheflow.parallel_utils.parallel_state import (
14
+ get_data_parallel_rank,
15
+ get_tensor_model_parallel_group,
16
+ get_tensor_model_parallel_rank,
17
+ get_tensor_model_parallel_world_size,
18
+)
19
+
20
+from .utils import (
21
+ split_tensor_into_1d_equal_chunks,
22
+ gather_split_1d_tensor,
23
+)
24
+
25
+from cacheflow.parallel_utils.utils import safely_set_viewless_tensor_data
26
+
27
+# Default name for the model parallel rng tracker.
28
+_MODEL_PARALLEL_RNG_TRACKER_NAME = 'model-parallel-rng'
29
+
30
+
31
+def _set_cuda_rng_state(new_state, device=-1):
32
+ """Sets the random number generator state of the current GPU.
33
+
34
+ Argumentss:
35
+ new_state (torch.ByteTensor): The desired state
36
+ This function is adapted from PyTorch repo (torch.cuda.set_rng_state)
37
+ with a single change: the input state is not cloned. Cloning caused
38
+ major performance issues for +4 GPU cases.
39
+ """
40
+ if hasattr(_C, '_cuda_setRNGState') and callable(_C._cuda_setRNGState):
41
+ # older PyTorch
42
+ def cb():
43
+ with device_ctx_manager(device):
44
+ _C._cuda_setRNGState(new_state)
45
+ else:
46
+ # newer PyTorch
47
+ if device == -1:
48
+ device = torch.device('cuda')
49
+ elif isinstance(device, str):
50
+ device = torch.device(device)
51
+ elif isinstance(device, int):
52
+ device = torch.device('cuda', device)
53
+
54
+ def cb():
55
+ idx = device.index
56
+ if idx is None:
57
+ idx = torch.cuda.current_device()
58
+ default_generator = torch.cuda.default_generators[idx]
59
+ default_generator.set_state(new_state)
60
+
61
+ _lazy_call(cb)
62
+
63
+
64
+
65
+class CudaRNGStatesTracker:
66
+ """Tracker for the cuda RNG states.
67
+
68
+ Using the `add` method, a cuda rng state is initialized based on
69
+ the input `seed` and is assigned to `name`. Later, by forking the
70
+ rng state, we can perform operations and return to our starting
71
+ cuda state.
72
+ """
73
+
74
+ def __init__(self):
75
+ # Map from a string name to the cuda rng state.
76
+ self.states_ = {}
77
+ # Seeds are just for book keeping and ensure no seed is set twice.
78
+ self.seeds_ = set()
79
+
80
+ def reset(self):
81
+ """Set to the initial state (no tracker)."""
82
+ self.states_ = {}
83
+ self.seeds_ = set()
84
+
85
+ def get_states(self):
86
+ """Get rng states. Copy the dictionary so we have direct
87
+ pointers to the states, not just a pointer to the dictionary."""
88
+ states = {}
89
+ for name in self.states_:
90
+ states[name] = self.states_[name]
91
+ return states
92
+
93
+ def set_states(self, states):
94
+ """Set the rng states. For efficiency purposes, we do not check
95
+ the size of seed for compatibility."""
96
+ self.states_ = states
97
+
98
+ def add(self, name, seed):
99
+ """Track the rng state."""
100
+ # Check seed is not already used.
101
+ if seed in self.seeds_:
102
+ raise Exception('seed {} already exists'.format(seed))
103
+ self.seeds_.add(seed)
104
+ # Check that state is not already defined.
105
+ if name in self.states_:
106
+ raise Exception('cuda rng state {} already exists'.format(name))
107
+ # Get the current rng state.
108
+ orig_rng_state = torch.cuda.get_rng_state()
109
+ # Set the new state and store it.
110
+ torch.cuda.manual_seed(seed)
111
+ self.states_[name] = torch.cuda.get_rng_state()
112
+ # Reset rng state to what it was.
113
+ _set_cuda_rng_state(orig_rng_state)
114
+
115
+ @contextlib.contextmanager
116
+ def fork(self, name=_MODEL_PARALLEL_RNG_TRACKER_NAME):
117
+ """Fork the cuda rng state, perform operations, and exit with
118
+ the original state."""
119
+ # Check if we have added the state
120
+ if name not in self.states_:
121
+ raise Exception('cuda rng state {} is not added'.format(name))
122
+ # Store current rng state.
123
+ orig_cuda_rng_state = torch.cuda.get_rng_state()
124
+ # Set rng state to the desired one
125
+ _set_cuda_rng_state(self.states_[name])
126
+ # Do the stuff we wanted to do.
127
+ try:
128
+ yield
129
+ finally:
130
+ # Update the current rng state for later use.
131
+ self.states_[name] = torch.cuda.get_rng_state()
132
+ # And set the state to the original state we started with.
133
+ _set_cuda_rng_state(orig_cuda_rng_state)
134
+
135
+
136
+# RNG tracker object.
137
+_CUDA_RNG_STATE_TRACKER = CudaRNGStatesTracker()
138
+
139
+
140
+def get_cuda_rng_tracker():
141
+ """Get cuda rng tracker."""
142
+ return _CUDA_RNG_STATE_TRACKER
143
+
144
+
145
+def model_parallel_cuda_manual_seed(seed):
146
+ """Initialize model parallel cuda seed.
147
+
148
+ This function should be called after the model parallel is
149
+ initialized. Also, no torch.cuda.manual_seed should be called
150
+ after this function. Basically, this is replacement for that
151
+ function.
152
+ Two set of RNG states are tracked:
153
+ default state: This is for data parallelism and is the same among a
154
+ set of model parallel GPUs but different across
155
+ different model paralle groups. This is used for
156
+ example for dropout in the non-tensor-model-parallel regions.
157
+ tensor-model-parallel state: This state is different among a set of model
158
+ parallel GPUs, but the same across data parallel
159
+ groups. This is used for example for dropout in
160
+ model parallel regions.
161
+ """
162
+ # 2718 is just for fun and any POSITIVE value will work.
163
+ offset = seed + 2718
164
+ tensor_model_parallel_seed = offset + get_tensor_model_parallel_rank()
165
+ # Data parallel gets the original seed.
166
+ data_parallel_seed = seed
167
+
168
+ _CUDA_RNG_STATE_TRACKER.reset()
169
+ # Set the default state.
170
+ torch.cuda.manual_seed(data_parallel_seed)
171
+ # and model parallel state.
172
+ _CUDA_RNG_STATE_TRACKER.add(_MODEL_PARALLEL_RNG_TRACKER_NAME,
173
+ tensor_model_parallel_seed)
174
+
175
+
176
+class CheckpointFunction(torch.autograd.Function):
177
+ """This function is adapted from torch.utils.checkpoint with
178
+ two main changes:
179
+ 1) torch.cuda.set_rng_state is replaced with `_set_cuda_rng_state`
180
+ 2) the states in the model parallel tracker are also properly
181
+ tracked/set/reset.
182
+ """
183
+ @staticmethod
184
+ def forward(ctx, run_function, distribute_saved_activations, *args):
185
+ ctx.run_function = run_function
186
+ ctx.distribute_saved_activations \
187
+ = distribute_saved_activations
188
+
189
+ # Copy the rng states.
190
+ ctx.fwd_cpu_rng_state = torch.get_rng_state()
191
+ ctx.fwd_cuda_rng_state = torch.cuda.get_rng_state()
192
+ ctx.fwd_cuda_rng_state_tracker = get_cuda_rng_tracker().get_states()
193
+
194
+ with torch.no_grad():
195
+ outputs = run_function(*args)
196
+
197
+ # Divide hidden states across model parallel group and only keep
198
+ # the chunk corresponding to the current rank.
199
+ if distribute_saved_activations:
200
+ ctx.input_0_shape = args[0].data.shape
201
+ safely_set_viewless_tensor_data(
202
+ args[0],
203
+ split_tensor_into_1d_equal_chunks(args[0].data, new_buffer=True))
204
+
205
+ # Store everything.
206
+ ctx.save_for_backward(*args)
207
+
208
+ return outputs
209
+
210
+ @staticmethod
211
+ def backward(ctx, *args):
212
+ if not torch.autograd._is_checkpoint_valid():
213
+ raise RuntimeError("Checkpointing is not compatible with .grad(), "
214
+ "please use .backward() if possible")
215
+ inputs = ctx.saved_tensors
216
+ if ctx.distribute_saved_activations:
217
+ safely_set_viewless_tensor_data(
218
+ inputs[0],
219
+ gather_split_1d_tensor(inputs[0].data).view(ctx.input_0_shape))
220
+
221
+ # Store the current states.
222
+ bwd_cpu_rng_state = torch.get_rng_state()
223
+ bwd_cuda_rng_state = torch.cuda.get_rng_state()
224
+ bwd_cuda_rng_state_tracker = get_cuda_rng_tracker().get_states()
225
+
226
+ # Set the states to what it used to be before the forward pass.
227
+ torch.set_rng_state(ctx.fwd_cpu_rng_state)
228
+ _set_cuda_rng_state(ctx.fwd_cuda_rng_state)
229
+ get_cuda_rng_tracker().set_states(ctx.fwd_cuda_rng_state_tracker)
230
+
231
+ # Compute the forward pass.
232
+ detached_inputs = detach_variable(inputs)
233
+ with torch.enable_grad():
234
+ outputs = ctx.run_function(*detached_inputs)
235
+
236
+ # Set the states back to what it was at the start of this function.
237
+ torch.set_rng_state(bwd_cpu_rng_state)
238
+ _set_cuda_rng_state(bwd_cuda_rng_state)
239
+ get_cuda_rng_tracker().set_states(bwd_cuda_rng_state_tracker)
240
+
241
+ if isinstance(outputs, torch.Tensor):
242
+ outputs = (outputs,)
243
+ torch.autograd.backward(outputs, args)
244
+ grads = tuple(inp.grad if isinstance(inp, torch.Tensor) else inp
245
+ for inp in detached_inputs)
246
+ return (None, None) + grads
247
+
248
+
249
+def checkpoint(function, distribute_saved_activations, *args):
250
+ """Checkpoint a model or part of the model.
251
+ This has been directly copied from torch.utils.checkpoint."""
252
+ return CheckpointFunction.apply(function,
253
+ distribute_saved_activations, *args)
cacheflow/parallel_utils/tensor_parallel/utils.py
new
+108
@@ -0,0 +1,108 @@
1
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2
+
3
+import torch
4
+from typing import List, Sequence
5
+
6
+from cacheflow.parallel_utils.utils import divide
7
+from cacheflow.parallel_utils import parallel_state
8
+
9
+def split_tensor_along_last_dim(
10
+ tensor: torch.Tensor,
11
+ num_partitions: int,
12
+ contiguous_split_chunks: bool = False,
13
+) -> List[torch.Tensor]:
14
+ """ Split a tensor along its last dimension.
15
+
16
+ Arguments:
17
+ tensor: input tensor.
18
+ num_partitions: number of partitions to split the tensor
19
+ contiguous_split_chunks: If True, make each chunk contiguous
20
+ in memory.
21
+
22
+ Returns:
23
+ A list of Tensors
24
+ """
25
+ # Get the size and dimension.
26
+ last_dim = tensor.dim() - 1
27
+ last_dim_size = divide(tensor.size()[last_dim], num_partitions)
28
+ # Split.
29
+ tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
30
+ # Note: torch.split does not create contiguous tensors by default.
31
+ if contiguous_split_chunks:
32
+ return tuple(chunk.contiguous() for chunk in tensor_list)
33
+
34
+ return tensor_list
35
+
36
+def split_tensor_into_1d_equal_chunks(tensor, new_buffer=False):
37
+ """ Break a tensor into equal 1D chunks across tensor parallel ranks.
38
+
39
+ Returns a Tensor or View with this rank's portion of the data.
40
+
41
+ Arguments:
42
+ tensor: The tensor to split
43
+
44
+ Keyword Arguments:
45
+ new_buffer (bool): If True, returns a new Tensor.
46
+ If False, returns a view into the existing Tensor.
47
+ Default is False
48
+
49
+ """
50
+ partition_size = torch.numel(tensor) // \
51
+ parallel_state.get_tensor_model_parallel_world_size()
52
+ start_index = partition_size * parallel_state.get_tensor_model_parallel_rank()
53
+ end_index = start_index + partition_size
54
+ if new_buffer:
55
+ data = torch.empty(partition_size, dtype=tensor.dtype,
56
+ device=torch.cuda.current_device(),
57
+ requires_grad=False)
58
+ data.copy_(tensor.view(-1)[start_index:end_index])
59
+ else:
60
+ data = tensor.view(-1)[start_index:end_index]
61
+ return data
62
+
63
+
64
+def gather_split_1d_tensor(tensor):
65
+ """ Opposite of split_tensor_into_1d_equal_chunks. Gather values from tensor
66
+ model parallel ranks.
67
+
68
+ Returns a new Tensor with the gathered data.
69
+
70
+ Arguments:
71
+ tensor: A Tensor or view of this rank's portion of the data.
72
+ """
73
+ numel_gathered = torch.numel(tensor) * \
74
+ parallel_state.get_tensor_model_parallel_world_size()
75
+ gathered = torch.empty(numel_gathered, dtype=tensor.dtype,
76
+ device=torch.cuda.current_device(),
77
+ requires_grad=False)
78
+ # TODO: This API is experimental in pytorch (as of Feb 2022) and
79
+ # this might break in future pytorch releases. We chose this API
80
+ # as opposed to torch.distributed.all_gather for efficiency reasons.
81
+ # This API calls directly NCCL all-gather versus the former does
82
+ # internal copies and can potentially cause slow down.
83
+ torch.distributed._all_gather_base(gathered, tensor,
84
+ group=parallel_state.get_tensor_model_parallel_group())
85
+ return gathered
86
+
87
+
88
+class VocabUtility:
89
+ """ Split the vocabulary into `world_size` chunks and return the first
90
+ and last index of the vocabulary belonging to the `rank`
91
+ partition: Note that indices in [fist, last)
92
+
93
+ """
94
+
95
+ @staticmethod
96
+ def vocab_range_from_per_partition_vocab_size(
97
+ per_partition_vocab_size: int, rank, world_size: int
98
+ ) -> Sequence[int]:
99
+ index_f = rank * per_partition_vocab_size
100
+ index_l = index_f + per_partition_vocab_size
101
+ return index_f, index_l
102
+
103
+ @staticmethod
104
+ def vocab_range_from_global_vocab_size(global_vocab_size: int, rank: int, world_size: int) -> Sequence[int]:
105
+ per_partition_vocab_size = divide(global_vocab_size, world_size)
106
+ return VocabUtility.vocab_range_from_per_partition_vocab_size(
107
+ per_partition_vocab_size, rank, world_size
108
+ )
cacheflow/parallel_utils/utils.py
new
+120
@@ -0,0 +1,120 @@
1
+"""Utility functions used throughout Megatron core"""
2
+from functools import reduce
3
+import operator
4
+
5
+import torch
6
+
7
+from cacheflow.parallel_utils import parallel_state
8
+
9
+
10
+def ensure_divisibility(numerator, denominator):
11
+ """Ensure that numerator is divisible by the denominator."""
12
+ assert numerator % denominator == 0, "{} is not divisible by {}".format(
13
+ numerator, denominator
14
+ )
15
+
16
+
17
+def divide(numerator, denominator):
18
+ """Ensure that numerator is divisible by the denominator and return
19
+ the division value."""
20
+ ensure_divisibility(numerator, denominator)
21
+ return numerator // denominator
22
+
23
+
24
+class GlobalMemoryBuffer:
25
+ """Global buffer to avoid dynamic memory allocations.
26
+ Caller should ensure that buffers of the same name
27
+ are not used concurrently."""
28
+
29
+ def __init__(self):
30
+ self.buffer = {}
31
+
32
+ def get_tensor(self, tensor_shape, dtype, name):
33
+ required_len = reduce(operator.mul, tensor_shape, 1)
34
+ if self.buffer.get((name, dtype), None) is None or \
35
+ self.buffer[(name, dtype)].numel() < required_len:
36
+ self.buffer[(name, dtype)] = \
37
+ torch.empty(required_len,
38
+ dtype=dtype,
39
+ device=torch.cuda.current_device(),
40
+ requires_grad=False)
41
+
42
+ return self.buffer[(name, dtype)][0:required_len].view(*tensor_shape)
43
+
44
+def _kernel_make_viewless_tensor(inp, requires_grad):
45
+ '''Make a viewless tensor.
46
+
47
+ View tensors have the undesirable side-affect of retaining a reference
48
+ to the originally-viewed tensor, even after manually setting the '.data'
49
+ field. This method creates a new tensor that links to the old tensor's
50
+ data, without linking the viewed tensor, referenced via the '._base'
51
+ field.
52
+ '''
53
+ out = torch.empty(
54
+ (1,),
55
+ dtype = inp.dtype,
56
+ device = inp.device,
57
+ requires_grad = requires_grad,
58
+ )
59
+ out.data = inp.data
60
+ return out
61
+
62
+class MakeViewlessTensor(torch.autograd.Function):
63
+ '''
64
+ Autograd function to make a viewless tensor.
65
+
66
+ This function should be used in cases where the computation graph needs
67
+ to be propagated, but we only want a viewless tensor (e.g.,
68
+ ParallelTransformer's hidden_states). Call this function by passing
69
+ 'keep_graph = True' to 'make_viewless_tensor()'.
70
+ '''
71
+ @staticmethod
72
+ def forward(ctx, inp, requires_grad):
73
+ return _kernel_make_viewless_tensor(inp, requires_grad)
74
+ @staticmethod
75
+ def backward(ctx, grad_output):
76
+ return grad_output, None
77
+
78
+def make_viewless_tensor(inp, requires_grad, keep_graph):
79
+ '''
80
+ Entry-point for creating viewless tensors.
81
+
82
+ This method should be used, rather than calling 'MakeViewlessTensor'
83
+ or '_kernel_make_viewless_tensor' directly. This method acts as a
84
+ switch for determining if an autograd function or a regular method
85
+ should be used to create the tensor.
86
+ '''
87
+
88
+ # return tensor as-is, if not a 'view'
89
+ if inp._base is None:
90
+ return inp
91
+
92
+ # create viewless tensor
93
+ if keep_graph:
94
+ return MakeViewlessTensor.apply(inp, requires_grad)
95
+ else:
96
+ return _kernel_make_viewless_tensor(inp, requires_grad)
97
+
98
+def assert_viewless_tensor(tensor, extra_msg = None):
99
+ '''Assert that a tensor is not a view (i.e., its '._base' field is
100
+ not set).'''
101
+ if isinstance(tensor, list):
102
+ [ assert_viewless_tensor(t) for t in tensor ]
103
+ return tensor
104
+ if not isinstance(tensor, torch.Tensor):
105
+ return tensor
106
+ assert tensor._base is None, (
107
+ "Ensure tensor._base is None before setting tensor.data or storing "
108
+ "tensor to memory buffer. Otherwise, a memory leak will occur (and "
109
+ "likely accumulate over iterations). %s"
110
+ ) % extra_msg
111
+ return tensor
112
+
113
+def safely_set_viewless_tensor_data(tensor, new_data_tensor):
114
+ '''Safely set tensor's '.data' field.
115
+
116
+ Check first that the tensor is viewless (i.e., '._base' not set). If not,
117
+ raise an exception.
118
+ '''
119
+ assert_viewless_tensor(tensor, extra_msg = "FYI, tensor._base has shape %s, and new_data_tensor has shape %s." % ("--" if tensor._base is None else tensor._base.shape, new_data_tensor.shape))
120
+ tensor.data = new_data_tensor
cacheflow/sequence.py
+6
@@ -158,3 +158,9 @@ class SequenceOutputs:
158
f'parent_seq_id={self.parent_seq_id}, '
159
f'output_token={self.output_token}), '
160
f'logprobs={self.logprobs}')
161
+
162
+ def __eq__(self, other: 'SequenceOutputs') -> bool:
163
+ return (self.seq_id == other.seq_id and
164
+ self.parent_seq_id == other.parent_seq_id and
165
+ self.output_token == other.output_token and
166
+ self.logprobs == other.logprobs)
cacheflow/utils.py
+17
@@ -1,4 +1,11 @@
1
import enum
2
+import random
3
+
4
+import numpy as np
5
+import torch
6
+
7
+from cacheflow.parallel_utils.parallel_state import model_parallel_is_initialized
8
+from cacheflow.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
9
10
11
class Device(enum.Enum):
@@ -18,3 +25,13 @@ class Counter:
25
26
def reset(self) -> None:
27
self.counter = 0
28
+
29
+def set_random_seed(seed: int):
30
+ random.seed(seed)
31
+ np.random.seed(seed)
32
+ torch.manual_seed(seed)
33
+ if torch.cuda.is_available():
34
+ torch.cuda.manual_seed_all(seed)
35
+
36
+ if model_parallel_is_initialized():
37
+ model_parallel_cuda_manual_seed(seed)
cacheflow/worker/cache_engine.py
+4
-6
@@ -11,7 +11,6 @@ class CacheEngine:
11
def __init__(
12
self,
13
worker_id: int,
14
- gpu_id: int,
14
num_layers: int,
15
num_heads: int,
16
head_size: int,
@@ -25,7 +24,6 @@ class CacheEngine:
24
f'head_size ({head_size}) must be a multiple of 16.')
25
26
self.worker_id = worker_id
28
- self.gpu_id = gpu_id
27
self.num_layers = num_layers
28
self.num_heads = num_heads
29
self.head_size = head_size
@@ -39,8 +37,8 @@ class CacheEngine:
37
self.cpu_cache = self.allocate_cpu_cache()
38
39
# Initialize the stream for caching operations.
42
- self.cache_stream = torch.cuda.Stream(device=gpu_id)
43
- assert self.cache_stream != torch.cuda.current_stream(device=gpu_id)
40
+ self.cache_stream = torch.cuda.Stream()
41
+ assert self.cache_stream != torch.cuda.current_stream()
42
# Initialize the events for stream synchronization.
43
self.events = [torch.cuda.Event() for _ in range(num_layers)]
44
@@ -69,12 +67,12 @@ class CacheEngine:
67
key_blocks = torch.empty(
68
size=(self.num_gpu_blocks, *key_block_shape),
69
dtype=self.dtype,
72
- device=self.gpu_id,
70
+ device="cuda",
71
)
72
value_blocks = torch.empty(
73
size=(self.num_gpu_blocks, *value_block_shape),
74
dtype=self.dtype,
77
- device=self.gpu_id,
75
+ device="cuda",
76
)
77
gpu_cache.append((key_blocks, value_blocks))
78
return gpu_cache
cacheflow/worker/controller.py
+42
-19
@@ -1,45 +1,62 @@
1
-from typing import Dict, List, Union
1
+from typing import Dict, List, Union, Tuple
2
+
3
+import ray
4
5
from cacheflow.master.scheduler import Scheduler
6
from cacheflow.sequence import SequenceGroupInputs
7
from cacheflow.worker.worker import Worker
8
9
10
+DeviceID = Tuple[int, str, int] # rank, node resource (node IP), device id
11
+
12
+
13
class Controller:
14
15
def __init__(
16
self,
12
- node_id: int,
13
- num_workers: int,
17
+ stage_id: int,
18
+ stage_devices: List[DeviceID],
19
+ world_size: int,
20
+ tensor_parallel_size: int,
21
+ pipeline_parallel_size: int,
22
+ distributed_init_method: str,
23
model_name: str,
24
block_size: int,
25
num_gpu_blocks: int,
26
num_cpu_blocks: int,
27
dtype: str,
28
seed: int,
29
+ model_path: str,
30
) -> None:
21
- self.node_id = node_id
22
- self.num_workers = num_workers
31
+ self.stage_id = stage_id
32
+ self.stage_devices = stage_devices
33
self.model_name = model_name
34
self.block_size = block_size
35
self.num_gpu_blocks = num_gpu_blocks
36
self.num_cpu_blocks = num_cpu_blocks
37
38
# Which pipeline stage is this node assigned to?
29
- self.is_first_stage = node_id == 0
39
+ self.is_first_stage = stage_id == 0
40
self.is_last_stage = False
41
42
self.workers: List[Worker] = []
33
- for i in range(num_workers):
34
- worker = Worker(
35
- worker_id=node_id + i,
36
- gpu_id=i,
43
+ for rank, node_resource, device_id in stage_devices:
44
+ worker_cls = ray.remote(num_cpus=0,
45
+ num_gpus=1,
46
+ resources={node_resource: 1e-5})(Worker)
47
+ worker = worker_cls.remote(
48
model_name=model_name,
49
block_size=block_size,
50
num_gpu_blocks=num_gpu_blocks,
51
num_cpu_blocks=num_cpu_blocks,
52
dtype=dtype,
53
seed=seed,
54
+ distributed_init_method=distributed_init_method,
55
+ rank=rank,
56
+ world_size=world_size,
57
+ tensor_parallel_size=tensor_parallel_size,
58
+ pipeline_parallel_size=pipeline_parallel_size,
59
+ model_path=model_path,
60
)
61
self.workers.append(worker)
62
@@ -57,15 +74,21 @@ class Controller:
74
blocks_to_swap_out: Dict[int, int],
75
blocks_to_copy: Dict[int, List[int]],
76
) -> None:
60
- # FIXME: Support tensor parallelism.
61
- assert len(self.workers) == 1
62
- worker = self.workers[0]
63
- output = worker.execute_stage(
64
- input_seq_groups,
65
- blocks_to_swap_in,
66
- blocks_to_swap_out,
67
- blocks_to_copy,
68
- )
77
+ futures = []
78
+ for worker in self.workers:
79
+ future = worker.execute_stage.remote(
80
+ input_seq_groups,
81
+ blocks_to_swap_in,
82
+ blocks_to_swap_out,
83
+ blocks_to_copy,
84
+ )
85
+ futures.append(future)
86
+
87
+ all_outputs = ray.get(futures)
88
+ # Make sure all workers have the same results.
89
+ output = all_outputs[0]
90
+ for other_output in all_outputs[1:]:
91
+ assert output == other_output
92
93
if self.is_last_stage:
94
self.next_node.post_step(output)
cacheflow/worker/worker.py
+51
-22
@@ -3,49 +3,58 @@ from typing import Dict, List, Tuple
3
import torch
4
5
from cacheflow.models import get_model
6
-from cacheflow.models import set_seed
6
from cacheflow.models import InputMetadata
7
from cacheflow.sampling_params import SamplingParams
8
from cacheflow.sequence import SequenceGroupInputs
9
from cacheflow.sequence import SequenceOutputs
10
from cacheflow.worker.cache_engine import CacheEngine
11
+from cacheflow.parallel_utils.parallel_state import (
12
+ initialize_model_parallel, get_tensor_model_parallel_world_size)
13
+from cacheflow.utils import set_random_seed
14
15
16
class Worker:
17
18
def __init__(
19
self,
18
- worker_id: int,
19
- gpu_id: int,
20
model_name: str,
21
block_size: int,
22
num_gpu_blocks: int,
23
num_cpu_blocks: int,
24
dtype: str,
25
seed: int,
26
+ distributed_init_method: str,
27
+ rank: int,
28
+ world_size: int,
29
+ model_path: str,
30
+ tensor_parallel_size: int = 1,
31
+ pipeline_parallel_size: int = 1,
32
) -> None:
27
- self.worker_id = worker_id
28
- self.gpu_id = gpu_id
33
+ self.init_distributed_environment(distributed_init_method,
34
+ rank,
35
+ world_size,
36
+ tensor_parallel_size,
37
+ pipeline_parallel_size)
38
+ self.worker_id = rank
39
self.block_size = block_size
30
-
31
- self.device = torch.device('cuda', index=gpu_id)
40
+ set_random_seed(seed)
41
42
# Initialize the model.
34
- # FIXME(woosuk): This is a hack.
35
- self.model = get_model(model_name, dtype=dtype).to(device=self.device)
43
+ self.model, self.dtype = get_model(model_name, dtype=dtype, path=model_path)
44
+ self.model = self.model.cuda()
45
+ tensor_model_parallel_world_size = (
46
+ get_tensor_model_parallel_world_size())
47
self.num_layers = self.model.config.num_hidden_layers
37
- self.num_heads = self.model.config.num_attention_heads
38
- self.head_size = self.model.config.hidden_size // self.num_heads
39
- self.dtype = self.model.dtype
48
+ assert self.model.config.num_attention_heads % tensor_model_parallel_world_size == 0
49
+ self.num_heads = self.model.config.num_attention_heads // tensor_model_parallel_world_size
50
+ self.head_size = self.model.config.hidden_size // (self.num_heads * tensor_model_parallel_world_size)
51
41
- # Set the seed.
42
- # We set the seed after initializing the model to ensure that
52
+ # We reset the seed after initializing the model to ensure that
53
# the random state is not affected by the model initialization.
44
- set_seed(seed)
54
+ set_random_seed(seed)
55
56
self.cache_engine = CacheEngine(
47
- worker_id=worker_id,
48
- gpu_id=gpu_id,
57
+ worker_id=self.worker_id,
58
num_layers=self.num_layers,
59
num_heads=self.num_heads,
60
head_size=self.head_size,
@@ -57,6 +66,26 @@ class Worker:
66
self.cache_events = self.cache_engine.events
67
self.gpu_cache = self.cache_engine.gpu_cache
68
69
+
70
+ def init_distributed_environment(self,
71
+ distributed_init_method: str,
72
+ rank: int,
73
+ world_size: int,
74
+ tensor_parallel_size: int = 1,
75
+ pipeline_parallel_size: int = 1) -> None:
76
+ """Initialize the distributed environment."""
77
+ torch.distributed.init_process_group(
78
+ backend='nccl',
79
+ init_method=distributed_init_method,
80
+ world_size=world_size,
81
+ rank=rank,
82
+ )
83
+ # A small all_reduce for warmup.
84
+ torch.distributed.all_reduce(torch.zeros(1).cuda())
85
+ initialize_model_parallel(tensor_parallel_size,
86
+ pipeline_parallel_size)
87
+
88
+
89
def prepare_inputs(
90
self,
91
input_seq_groups: List[SequenceGroupInputs],
@@ -142,18 +171,18 @@ class Worker:
171
172
# Convert to tensors.
173
tokens_tensor = torch.tensor(
145
- input_tokens, dtype=torch.long, device=self.device)
174
+ input_tokens, dtype=torch.long, device='cuda')
175
positions_tensor = torch.tensor(
147
- input_positions, dtype=torch.long, device=self.device)
176
+ input_positions, dtype=torch.long, device='cuda')
177
slot_mapping_tensor = torch.tensor(
149
- slot_mapping, dtype=torch.int, device=self.device)
178
+ slot_mapping, dtype=torch.int, device='cuda')
179
context_lens_tensor = torch.tensor(
151
- context_lens, dtype=torch.int, device=self.device)
180
+ context_lens, dtype=torch.int, device='cuda')
181
padded_block_tables = [
182
_pad_to_max(block_table, max_num_blocks_per_seq)
183
for block_table in generation_block_tables]
184
block_tables_tensor = torch.tensor(
156
- padded_block_tables, dtype=torch.int, device=self.device)
185
+ padded_block_tables, dtype=torch.int, device='cuda')
186
187
input_metadata = InputMetadata(
188
seq_groups=seq_groups,
server.py
+115
-23
@@ -1,30 +1,99 @@
1
import argparse
2
-from typing import List
2
+import random
3
+from typing import List, Tuple, Dict
4
+
5
+import ray
6
7
from cacheflow.master.frontend import Frontend
8
from cacheflow.master.scheduler import Scheduler
9
from cacheflow.models import get_memory_analyzer
7
-from cacheflow.worker.controller import Controller
8
-
9
-parser = argparse.ArgumentParser(description='CacheFlow server')
10
-parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
11
-parser.add_argument('--num-nodes', type=int, default=1, help='number of nodes')
12
-parser.add_argument('--num-workers', type=int, default=1, help='number of workers per node')
13
-parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='token block size')
14
-# NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
15
-parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
16
-# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
17
-parser.add_argument('--seed', type=int, default=0, help='random seed')
18
-parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
19
-parser.add_argument('--max-batch-size', type=int, default=2560, help='maximum number of batched tokens')
20
-args = parser.parse_args()
21
-
22
-
23
-def main():
10
+from cacheflow.worker.controller import Controller, DeviceID
11
+
12
+
13
+def initialize_ray_cluster(
14
+ address: str = 'auto',
15
+ pipeline_parallel_size: int = 1,
16
+ tensor_parallel_size: int = 1,
17
+) -> Tuple[int, int, str, List[List[DeviceID]]]:
18
+ # Connect to a ray cluster.
19
+ ray.init(address=address)
20
+
21
+ # Assume we have a uniform cluster that each node has the same number of
22
+ # GPUs for now.
23
+ valid_node_resources = []
24
+ num_devices_per_node = None
25
+ for node in ray.nodes():
26
+ if (not node['Alive']) or node['Resources']['GPU'] <= 0:
27
+ continue
28
+ if num_devices_per_node is None:
29
+ num_devices_per_node = node['Resources']['GPU']
30
+ else:
31
+ assert num_devices_per_node == node['Resources']['GPU'], (
32
+ "The number of GPUs per node is not uniform.")
33
+ for key in node['Resources']:
34
+ if key.startswith('node:'):
35
+ valid_node_resources.append(key)
36
+
37
+ num_nodes = len(valid_node_resources)
38
+
39
+ assert (pipeline_parallel_size * tensor_parallel_size
40
+ <= num_nodes * num_devices_per_node), (
41
+ "The number of required GPUs exceeds the total number of "
42
+ "available GPUs.")
43
+ if tensor_parallel_size >= num_devices_per_node:
44
+ assert tensor_parallel_size % num_devices_per_node == 0, (
45
+ "The number of tensor parallelism is not divisible by the "
46
+ "number of GPUs per node.")
47
+ else:
48
+ assert num_devices_per_node % tensor_parallel_size == 0, (
49
+ "The number of GPUs per node is not divisible by the number "
50
+ "of tensor parallelism.")
51
+
52
+ # Assign GPUs to pipeline stages.
53
+ rank = 0
54
+ current_node_id = 0
55
+ current_device_id = 0
56
+ distributed_init_method = None
57
+ all_stage_devices = []
58
+
59
+ for i in range(pipeline_parallel_size):
60
+ stage_devices = []
61
+ for j in range(tensor_parallel_size):
62
+ node_resource = valid_node_resources[current_node_id]
63
+ stage_devices.append((rank, node_resource, current_device_id))
64
+ if distributed_init_method is None:
65
+ ip = node_resource.split("node:")[-1]
66
+ port = random.randint(10000, 20000)
67
+ distributed_init_method = f"tcp://{ip}:{port}"
68
+ rank += 1
69
+ current_device_id += 1
70
+ if current_device_id >= num_devices_per_node:
71
+ current_node_id += 1
72
+ current_device_id = 0
73
+ all_stage_devices.append(stage_devices)
74
+
75
+ return (num_nodes, num_devices_per_node, distributed_init_method,
76
+ all_stage_devices)
77
+
78
+
79
+def main(args: argparse.Namespace):
80
+ # TODO(zhuohan): Support pipeline parallelism.
81
+ assert args.pipeline_parallel_size == 1, (
82
+ 'Pipeline parallelism is not supported yet.')
83
+
84
+ (num_nodes, num_devices_per_node, distributed_init_method,
85
+ all_stage_devices) = (
86
+ initialize_ray_cluster(
87
+ pipeline_parallel_size=args.pipeline_parallel_size,
88
+ tensor_parallel_size=args.tensor_parallel_size))
89
+
90
+ world_size = args.pipeline_parallel_size * args.tensor_parallel_size
91
+
92
memory_analyzer = get_memory_analyzer(
93
model_name=args.model,
94
block_size=args.block_size,
95
dtype=args.dtype,
96
+ tensor_parallel_size=args.tensor_parallel_size,
97
)
98
num_gpu_blocks = memory_analyzer.get_max_num_gpu_blocks(
99
max_num_batched_tokens=args.max_batch_size)
@@ -32,18 +101,23 @@ def main():
101
swap_space=args.swap_space)
102
print(f'# GPU blocks: {num_gpu_blocks}, # CPU blocks: {num_cpu_blocks}')
103
35
- # Create a controller for each node.
104
+ # Create a controller for each pipeline stage.
105
controllers: List[Controller] = []
37
- for i in range(args.num_nodes):
106
+ for i in range(args.pipeline_parallel_size):
107
controller = Controller(
39
- node_id=i,
40
- num_workers=args.num_workers,
108
+ stage_id=i,
109
+ stage_devices=all_stage_devices[i],
110
+ world_size=world_size,
111
+ pipeline_parallel_size=args.pipeline_parallel_size,
112
+ tensor_parallel_size=args.tensor_parallel_size,
113
+ distributed_init_method=distributed_init_method,
114
model_name=args.model,
115
block_size=args.block_size,
116
num_gpu_blocks=num_gpu_blocks,
117
num_cpu_blocks=num_cpu_blocks,
118
dtype=args.dtype,
119
seed=args.seed,
120
+ model_path=args.model_path,
121
)
122
controllers.append(controller)
123
@@ -83,4 +157,22 @@ def main():
157
158
159
if __name__ == '__main__':
86
- main()
160
+ parser = argparse.ArgumentParser(description='CacheFlow server')
161
+ # Model arguments
162
+ parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
163
+ parser.add_argument('--model-path', type=str, default='~/.cacheflow/model_weights',
164
+ help='model path to download and load the weights')
165
+ # Parallel arguments
166
+ parser.add_argument('--pipeline-parallel-size', type=int, default=1, help='number of pipeline stages')
167
+ parser.add_argument('--tensor-parallel-size', type=int, default=1, help='number of tensor parallel replicas')
168
+ # KV cache arguments
169
+ parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='token block size')
170
+ # NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
171
+ parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
172
+ # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
173
+ parser.add_argument('--seed', type=int, default=0, help='random seed')
174
+ parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
175
+ parser.add_argument('--max-batch-size', type=int, default=2560, help='maximum number of batched tokens')
176
+ args = parser.parse_args()
177
+
178
+ main(args)