@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2f49f15585
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Support tensor parallel (#2)

Zhuohan Li committed Mar 22, 2023 at 04:45 UTC 2f49f155858faaf82bfd076a821497e41e961658
24 files changed +2482 -176
README.md
+2 -1
@@ -11,5 +11,6 @@ pip install -e .
11 ## Run
12
13 ```bash
14 -python server.py
14 +ray start --head
15 +python server.py [--tensor-parallel-size <N>]
16 ```
cacheflow/models/__init__.py
-2
@@ -1,12 +1,10 @@
1 from cacheflow.models.input_metadata import InputMetadata
2 from cacheflow.models.model_utils import get_memory_analyzer
3 from cacheflow.models.model_utils import get_model
4 -from cacheflow.models.utils import set_seed
4
5
6 __all__ = [
7 'InputMetadata',
8 'get_memory_analyzer',
9 'get_model',
11 - 'set_seed',
10 ]
cacheflow/models/attention.py
+1 -1
@@ -112,7 +112,7 @@ class OPTCacheFlowAttention(nn.Module):
112 output[:num_prompt_tokens],
113 query[:num_prompt_tokens],
114 key[:num_prompt_tokens],
115 - value[:num_prompt_tokens],
115 + value[:num_prompt_tokens],
116 input_metadata.prompt_lens,
117 )
118
cacheflow/models/input_metadata.py
+5 -1
@@ -43,4 +43,8 @@ class InputMetadata:
43 f'num_generation_tokens={self.num_generation_tokens}, '
44 f'num_valid_tokens={self.num_valid_tokens}, '
45 f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
46 - f'max_context_len={self.max_context_len})')
46 + f'max_context_len={self.max_context_len}), '
47 + f'prompt_lens={self.prompt_lens}, '
48 + f'slot_mapping={self.slot_mapping}, '
49 + f'context_lens={self.context_lens}, '
50 + f'block_tables={self.block_tables})')
cacheflow/models/memory_analyzer.py
+18 -16
@@ -31,12 +31,13 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
31 model_name: str,
32 block_size: int,
33 dtype: torch.dtype,
34 + tensor_parallel_size: int,
35 ) -> None:
36 self.model_name = model_name
37 self.block_size = block_size
38 self.dtype = dtype
39 + self.tensor_parallel_size = tensor_parallel_size
40
39 - # TODO(woosuk): Support tensor parallelism.
41 config = AutoConfig.from_pretrained(model_name)
42 self.num_layers = config.num_hidden_layers
43 self.hidden_size = config.hidden_size
@@ -48,26 +49,25 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
49 self.max_position = config.max_position_embeddings
50
51 def _get_param_size(self) -> int:
51 - # TODO(woosuk): Support tensor parallelism.
52 - word_embedding = self.vocab_size * self.embedding_size
52 + word_embedding = self.vocab_size * self.embedding_size // self.tensor_parallel_size
53 if self.embedding_size != self.vocab_size:
54 # Project in/out.
55 word_embedding += 2 * self.embedding_size * self.vocab_size
56 position_embedding = self.max_position * self.hidden_size
57
58 ln1 = 2 * self.hidden_size
59 - q = self.hidden_size * self.hidden_size + self.hidden_size
60 - k = self.hidden_size * self.hidden_size + self.hidden_size
61 - v = self.hidden_size * self.hidden_size + self.hidden_size
62 - out = self.hidden_size * self.hidden_size + self.hidden_size
59 + q = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
60 + k = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
61 + v = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
62 + out = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
63 mha = ln1 + q + k + v + out
64
65 ln2 = 2 * self.hidden_size
66 - ffn1 = self.hidden_size * self.ffn_size + self.ffn_size
67 - ffn2 = self.ffn_size * self.hidden_size + self.hidden_size
66 + ffn1 = self.hidden_size * self.ffn_size // self.tensor_parallel_size + self.ffn_size
67 + ffn2 = self.ffn_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
68 ffn = ln2 + ffn1 + ffn2
69
70 - total = (word_embedding + position_embedding +
70 + total = (word_embedding + position_embedding +
71 self.num_layers * (mha + ffn))
72 dtype_size = get_dtype_size(self.dtype)
73 return dtype_size * total
@@ -76,15 +76,17 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
76 self,
77 max_num_batched_tokens: int,
78 ) -> int:
79 - # TODO(woosuk): Support tensor parallelism.
79 # NOTE: We approxmiately calculate the maximum activation size by
81 - # 1) estimating the maximum activation tensor size during inference, and
82 - # 2) multiplying it by 4.
80 + # estimating
81 + # 1) the maximum activation tensor size during inference
82 + # 2) the residual tensor size during inference
83 # Here, we assume that FlashAttention is used and
84 # thus the attention maps are never materialized in GPU DRAM.
85 - qkv = 3 * (max_num_batched_tokens * self.hidden_size)
86 - ffn = max_num_batched_tokens * self.ffn_size
87 - max_act = 4 * max(qkv, ffn)
85 + residual = max_num_batched_tokens * self.hidden_size
86 + qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
87 + ffn = max_num_batched_tokens * self.ffn_size // self.tensor_parallel_size
88 + # Double the activation size for input and output.
89 + max_act = 2 * (max(qkv, ffn) + residual)
90 dtype_size = get_dtype_size(self.dtype)
91 return dtype_size * max_act
92
cacheflow/models/model_utils.py
+16 -6
@@ -1,7 +1,9 @@
1 from typing import Union
2
3 +import numpy as np
4 import torch
5 import torch.nn as nn
6 +from transformers import AutoConfig
7
8 from cacheflow.models.memory_analyzer import CacheFlowMemoryAnalyzer
9 from cacheflow.models.memory_analyzer import OPTMemoryAnalyzer
@@ -21,13 +23,20 @@ _MEMORY_ANALYZERS = {
23 def get_model(
24 model_name: str,
25 dtype: Union[torch.dtype, str],
26 + path: str,
27 ) -> nn.Module:
28 torch_dtype = get_torch_dtype(dtype)
26 - for model_class, hf_model in _MODELS.items():
27 - if model_class in model_name:
28 - model = hf_model.from_pretrained(
29 - model_name, torch_dtype=torch_dtype)
30 - return model.eval()
29 + torch.set_default_dtype(torch_dtype)
30 + config = AutoConfig.from_pretrained(model_name)
31 + for model_class_name, model_class in _MODELS.items():
32 + if model_class_name in model_name:
33 + # Download model weights if it's not cached.
34 + weights_dir = model_class.download_weights(model_name, path=path)
35 + # Create a model instance.
36 + model = model_class(config)
37 + # Load the weights from the cached or downloaded files.
38 + model.load_weights(weights_dir)
39 + return model.eval(), torch_dtype
40 raise ValueError(f'Unsupported model name: {model_name}')
41
42
@@ -35,10 +44,11 @@ def get_memory_analyzer(
44 model_name: str,
45 block_size: int,
46 dtype: Union[torch.dtype, str],
47 + tensor_parallel_size: int = 1,
48 ) -> CacheFlowMemoryAnalyzer:
49 torch_dtype = get_torch_dtype(dtype)
50 for model_class, memory_analyzer in _MEMORY_ANALYZERS.items():
51 if model_class in model_name:
52 return memory_analyzer(
43 - model_name, block_size, torch_dtype)
53 + model_name, block_size, torch_dtype, tensor_parallel_size)
54 raise ValueError(f'Unsupported model name: {model_name}')
cacheflow/models/opt.py
+131 -70
@@ -1,14 +1,24 @@
1 """1D OPT model compatible with HuggingFace weights."""
2 +import os
3 +import glob
4 +import filelock
5 +from tqdm import tqdm
6 from typing import Dict, List, Optional, Tuple
7
8 +import numpy as np
9 import torch
10 from torch import nn
11 from transformers import OPTConfig
7 -from transformers import PreTrainedModel
12 +from huggingface_hub import snapshot_download
13
14 from cacheflow.models import InputMetadata
15 from cacheflow.models.attention import OPTCacheFlowAttention
16 from cacheflow.models.sample import Sampler
17 +from cacheflow.parallel_utils.parallel_state import (
18 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
19 +from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
20 + ColumnParallelLinear,
21 + RowParallelLinear)
22 from cacheflow.sequence import SequenceOutputs
23
24 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -36,15 +46,26 @@ class OPTAttention(nn.Module):
46 ) -> None:
47 super().__init__()
48 self.embed_dim = embed_dim
39 - self.num_heads = num_heads
40 - self.head_dim = embed_dim // num_heads
49 + tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
50 + total_num_heads = num_heads
51 + assert num_heads % tensor_model_parallel_world_size == 0
52 + self.num_heads = total_num_heads // tensor_model_parallel_world_size
53 + self.head_dim = embed_dim // total_num_heads
54 self.scaling = self.head_dim**-0.5
55
56 # TODO(woosuk): Fuse the three linear layers into one QKV linear layer.
44 - self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
45 - self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
46 - self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
47 - self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
57 + self.k_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
58 + gather_output=False,
59 + perform_initialization=False)
60 + self.v_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
61 + gather_output=False,
62 + perform_initialization=False)
63 + self.q_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
64 + gather_output=False,
65 + perform_initialization=False)
66 + self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
67 + input_is_parallel=True,
68 + perform_initialization=False)
69
70 self.attn = OPTCacheFlowAttention(scale=self.scaling)
71
@@ -55,13 +76,13 @@ class OPTAttention(nn.Module):
76 input_metadata: InputMetadata,
77 cache_event: Optional[torch.cuda.Event],
78 ) -> torch.Tensor:
58 - q = self.q_proj(hidden_states)
59 - k = self.k_proj(hidden_states)
60 - v = self.v_proj(hidden_states)
79 + q, _ = self.q_proj(hidden_states)
80 + k, _ = self.k_proj(hidden_states)
81 + v, _ = self.v_proj(hidden_states)
82 key_cache, value_cache = kv_cache
83 attn_output = self.attn(
84 q, k, v, key_cache, value_cache, input_metadata, cache_event)
64 - output = self.out_proj(attn_output)
85 + output, _ = self.out_proj(attn_output)
86 return output
87
88
@@ -69,6 +90,7 @@ class OPTDecoderLayer(nn.Module):
90
91 def __init__(self, config: OPTConfig):
92 super().__init__()
93 + self.config = config
94 self.embed_dim = config.hidden_size
95 self.self_attn = OPTAttention(
96 embed_dim=self.embed_dim,
@@ -81,9 +103,16 @@ class OPTDecoderLayer(nn.Module):
103
104 self.self_attn_layer_norm = nn.LayerNorm(
105 self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
84 - self.fc1 = nn.Linear(self.embed_dim, config.ffn_dim, bias=config.enable_bias)
85 - self.fc2 = nn.Linear(config.ffn_dim, self.embed_dim, bias=config.enable_bias)
86 - self.final_layer_norm = nn.LayerNorm(self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
106 + self.fc1 = ColumnParallelLinear(self.embed_dim, config.ffn_dim,
107 + bias=config.enable_bias,
108 + gather_output=False,
109 + perform_initialization=False)
110 + self.fc2 = RowParallelLinear(config.ffn_dim, self.embed_dim,
111 + bias=config.enable_bias,
112 + input_is_parallel=True,
113 + perform_initialization=False)
114 + self.final_layer_norm = nn.LayerNorm(
115 + self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
116
117 def forward(
118 self,
@@ -112,9 +141,9 @@ class OPTDecoderLayer(nn.Module):
141 # 125m, 1.7B, ..., 175B applies layer norm BEFORE attention
142 if self.do_layer_norm_before:
143 hidden_states = self.final_layer_norm(hidden_states)
115 - hidden_states = self.fc1(hidden_states)
144 + hidden_states, _ = self.fc1(hidden_states)
145 hidden_states = self.activation_fn(hidden_states)
117 - hidden_states = self.fc2(hidden_states)
146 + hidden_states, _ = self.fc2(hidden_states)
147 hidden_states = residual + hidden_states
148 # 350m applies layer norm AFTER attention
149 if not self.do_layer_norm_before:
@@ -122,29 +151,23 @@ class OPTDecoderLayer(nn.Module):
151 return hidden_states
152
153
125 -class OPTPreTrainedModel(PreTrainedModel):
126 - config_class = OPTConfig
127 - base_model_prefix = "model"
128 - supports_gradient_checkpointing = True
129 - _no_split_modules = ["OPTDecoderLayer"]
130 - _keys_to_ignore_on_load_unexpected = [r"decoder\.version"]
131 -
132 - def _init_weights(self, module) -> None:
133 - del module # unused
134 - return
135 -
136 -
137 -class OPTDecoder(OPTPreTrainedModel):
154 +class OPTDecoder(nn.Module):
155
156 def __init__(self, config: OPTConfig):
140 - super().__init__(config)
157 + super().__init__()
158 + self.config = config
159 self.padding_idx = config.pad_token_id
160 self.max_target_positions = config.max_position_embeddings
161 self.vocab_size = config.vocab_size
162
145 - self.embed_tokens = nn.Embedding(config.vocab_size, config.word_embed_proj_dim, self.padding_idx)
146 - self.embed_positions = OPTLearnedPositionalEmbedding(config.max_position_embeddings, config.hidden_size)
163 + self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
164 + config.word_embed_proj_dim,
165 + perform_initialization=False)
166 + # Positional embeddings are replicated (not sharded).
167 + self.embed_positions = OPTLearnedPositionalEmbedding(
168 + config.max_position_embeddings, config.hidden_size)
169
170 + # Project out & in will be replicated if they exist.
171 if config.word_embed_proj_dim != config.hidden_size:
172 self.project_out = nn.Linear(config.hidden_size, config.word_embed_proj_dim, bias=False)
173 else:
@@ -167,9 +190,6 @@ class OPTDecoder(OPTPreTrainedModel):
190
191 self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)])
192
170 - # Initialize weights and apply final processing
171 - self.post_init()
172 -
193 def forward(
194 self,
195 input_ids: torch.LongTensor,
@@ -200,13 +220,11 @@ class OPTDecoder(OPTPreTrainedModel):
220 return hidden_states
221
222
203 -class OPTModel(OPTPreTrainedModel):
223 +class OPTModel(nn.Module):
224
225 def __init__(self, config: OPTConfig):
206 - super().__init__(config)
226 + super().__init__()
227 self.decoder = OPTDecoder(config)
208 - # Initialize weights and apply final processing
209 - self.post_init()
228
229 def forward(
230 self,
@@ -220,41 +238,17 @@ class OPTModel(OPTPreTrainedModel):
238 input_ids, positions, kv_caches, input_metadata, cache_events)
239
240
223 -class OPTForCausalLM(OPTPreTrainedModel):
224 - _keys_to_ignore_on_load_missing = [r"lm_head.weight"]
241 +class OPTForCausalLM(nn.Module):
242
243 def __init__(self, config):
227 - super().__init__(config)
244 + super().__init__()
245 + self.config = config
246 self.model = OPTModel(config)
229 - # the lm_head weight is automatically tied to the embed tokens weight
230 - self.lm_head = nn.Linear(config.word_embed_proj_dim, config.vocab_size, bias=False)
247 + # TODO(zhuohan): create a new weight after implementing pipeline
248 + # parallelism
249 + self.lm_head_weight = self.model.decoder.embed_tokens.weight
250 self.sampler = Sampler()
251
233 - # Initialize weights and apply final processing
234 - self.post_init()
235 -
236 - # NOTE(woosuk): While the following methods are not called in the model code,
237 - # they may be internally used by the transformers library.
238 - # For example, tie_weights() does not work without these methods.
239 - # Thus, do not delete these methods.
240 - def get_input_embeddings(self):
241 - return self.model.decoder.embed_tokens
242 -
243 - def set_input_embeddings(self, value):
244 - self.model.decoder.embed_tokens = value
245 -
246 - def get_output_embeddings(self):
247 - return self.lm_head
248 -
249 - def set_output_embeddings(self, new_embeddings):
250 - self.lm_head = new_embeddings
251 -
252 - def set_decoder(self, decoder):
253 - self.model.decoder = decoder
254 -
255 - def get_decoder(self):
256 - return self.model.decoder
257 -
252 def forward(
253 self,
254 input_ids: torch.LongTensor,
@@ -266,5 +260,72 @@ class OPTForCausalLM(OPTPreTrainedModel):
260 hidden_states = self.model(
261 input_ids, positions, kv_caches, input_metadata, cache_events)
262 next_tokens = self.sampler(
269 - self.lm_head.weight, hidden_states, input_metadata)
263 + self.lm_head_weight, hidden_states, input_metadata)
264 return next_tokens
265 +
266 + _column_parallel_weights = ["embed_tokens.weight",
267 + "q_proj.weight", "k_proj.weight",
268 + "v_proj.weight", "fc1.weight"]
269 + _column_parallel_biases = ["q_proj.bias", "k_proj.bias",
270 + "v_proj.bias", "fc1.bias"]
271 + _row_parallel_weights = ["out_proj.weight", "fc2.weight"]
272 +
273 + def load_weights(self, weights_path: str):
274 + tensor_model_parallel_rank = get_tensor_model_parallel_rank()
275 + state_dict = self.state_dict()
276 + for name, param in state_dict.items():
277 + if "lm_head_weight" in name:
278 + continue
279 + loaded_weight = torch.from_numpy(np.load(os.path.join(weights_path,
280 + name)))
281 + for p in (self._column_parallel_weights
282 + + self._column_parallel_biases):
283 + if p in name:
284 + shard_size = param.shape[0]
285 + loaded_weight = loaded_weight[
286 + shard_size * tensor_model_parallel_rank
287 + :shard_size * (tensor_model_parallel_rank + 1)]
288 + break
289 + for p in self._row_parallel_weights:
290 + if p in name:
291 + shard_size = param.shape[1]
292 + loaded_weight = loaded_weight[
293 + :,
294 + shard_size * tensor_model_parallel_rank
295 + :shard_size * (tensor_model_parallel_rank + 1)]
296 + break
297 +
298 + assert param.shape == loaded_weight.shape
299 + param.data.copy_(loaded_weight)
300 +
301 + @staticmethod
302 + def download_weights(model_name: str, path: str):
303 + path = os.path.join(path, f"{model_name}-np")
304 + path = os.path.abspath(os.path.expanduser(path))
305 + os.makedirs(path, exist_ok=True)
306 + lock_path = os.path.join(path, "file_lock")
307 + lock = filelock.FileLock(lock_path)
308 +
309 + with lock:
310 + test_weight_path = os.path.join(
311 + path, "model.decoder.embed_positions.weight")
312 + if os.path.exists(test_weight_path):
313 + return path
314 +
315 + folder = snapshot_download(model_name, allow_patterns="*.bin",
316 + cache_dir=os.path.join(path, "cache"))
317 + bin_files = glob.glob(os.path.join(folder, "*.bin"))
318 +
319 + if "/" in model_name:
320 + model_name = model_name.split("/")[1].lower()
321 +
322 + for bin_file in tqdm(bin_files, desc="Convert format"):
323 + state = torch.load(bin_file)
324 + for name, param in tqdm(state.items(), leave=False):
325 + if name.startswith("decoder."):
326 + name = "model." + name
327 + param_path = os.path.join(path, name)
328 + with open(param_path, "wb") as f:
329 + np.save(f, param.cpu().detach().numpy())
330 +
331 + return path
cacheflow/models/sample.py
+2 -1
@@ -6,7 +6,7 @@ import torch.nn as nn
6 from cacheflow.models import InputMetadata
7 from cacheflow.sampling_params import SamplingParams
8 from cacheflow.sequence import SequenceOutputs
9 -
9 +from cacheflow.parallel_utils.tensor_parallel import gather_from_tensor_model_parallel_region
10
11 class Sampler(nn.Module):
12
@@ -24,6 +24,7 @@ class Sampler(nn.Module):
24
25 # Get the logits for the next tokens.
26 logits = torch.matmul(hidden_states, embedding.t())
27 + logits = gather_from_tensor_model_parallel_region(logits)
28
29 # Apply temperature scaling.
30 temperatures = _get_temperatures(input_metadata)
cacheflow/models/utils.py
-8
@@ -27,14 +27,6 @@ def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
27 return torch.tensor([], dtype=torch_dtype).element_size()
28
29
30 -def set_seed(seed: int) -> None:
31 - random.seed(seed)
32 - np.random.seed(seed)
33 - torch.manual_seed(seed)
34 - if torch.cuda.is_available():
35 - torch.cuda.manual_seed_all(seed)
36 -
37 -
30 def get_gpu_memory(gpu: int = 0) -> int:
31 return torch.cuda.get_device_properties(gpu).total_memory
32
cacheflow/parallel_utils/README.md new
+1
@@ -0,0 +1 @@
1 +The files in this folder are ported from [Megatron-LM](https://github.com/NVIDIA/Megatron-LM/tree/main/megatron/core). We only keep the codes that are used in inference.
\ No newline at end of file
cacheflow/parallel_utils/__init__.py new
+12
@@ -0,0 +1,12 @@
1 +import cacheflow.parallel_utils.parallel_state
2 +import cacheflow.parallel_utils.tensor_parallel
3 +import cacheflow.parallel_utils.utils
4 +
5 +# Alias parallel_state as mpu, its legacy name
6 +mpu = parallel_state
7 +
8 +__all__ = [
9 + "parallel_state",
10 + "tensor_parallel",
11 + "utils",
12 +]
cacheflow/parallel_utils/parallel_state.py new
+522
@@ -0,0 +1,522 @@
1 +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2 +
3 +"""Model and data parallel groups."""
4 +
5 +import torch
6 +from typing import Optional
7 +
8 +from .utils import GlobalMemoryBuffer
9 +
10 +# Intra-layer model parallel group that the current rank belongs to.
11 +_TENSOR_MODEL_PARALLEL_GROUP = None
12 +# Inter-layer model parallel group that the current rank belongs to.
13 +_PIPELINE_MODEL_PARALLEL_GROUP = None
14 +# Model parallel group (both intra- and pipeline) that the current rank belongs to.
15 +_MODEL_PARALLEL_GROUP = None
16 +# Embedding group.
17 +_EMBEDDING_GROUP = None
18 +# Position embedding group.
19 +_POSITION_EMBEDDING_GROUP = None
20 +# Data parallel group that the current rank belongs to.
21 +_DATA_PARALLEL_GROUP = None
22 +
23 +_VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = None
24 +_VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
25 +_PIPELINE_MODEL_PARALLEL_SPLIT_RANK = None
26 +
27 +# These values enable us to change the mpu sizes on the fly.
28 +_MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = None
29 +_MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
30 +_MPU_TENSOR_MODEL_PARALLEL_RANK = None
31 +_MPU_PIPELINE_MODEL_PARALLEL_RANK = None
32 +
33 +# A list of ranks that have a copy of the embedding.
34 +_EMBEDDING_GLOBAL_RANKS = None
35 +
36 +# A list of ranks that have a copy of the position embedding.
37 +_POSITION_EMBEDDING_GLOBAL_RANKS = None
38 +
39 +# A list of global ranks for each pipeline group to ease calculation of the source
40 +# rank when broadcasting from the first or last pipeline stage.
41 +_PIPELINE_GLOBAL_RANKS = None
42 +
43 +# A list of global ranks for each data parallel group to ease calculation of the source
44 +# rank when broadcasting weights from src to all other data parallel ranks
45 +_DATA_PARALLEL_GLOBAL_RANKS = None
46 +
47 +# Memory buffers to avoid dynamic memory allocation
48 +_GLOBAL_MEMORY_BUFFER = None
49 +
50 +
51 +def initialize_model_parallel(
52 + tensor_model_parallel_size: int = 1,
53 + pipeline_model_parallel_size: int = 1,
54 + virtual_pipeline_model_parallel_size: Optional[int] = None,
55 + pipeline_model_parallel_split_rank: Optional[int] = None,
56 +) -> None:
57 + """
58 + Initialize model data parallel groups.
59 +
60 + Arguments:
61 + tensor_model_parallel_size: number of GPUs used for tensor model parallelism.
62 + pipeline_model_parallel_size: number of GPUs used for pipeline model parallelism.
63 + virtual_pipeline_model_parallel_size: number of virtual stages (interleaved
64 + pipeline).
65 + pipeline_model_parallel_split_rank: for models with both encoder and decoder,
66 + rank in pipeline with split point.
67 +
68 + Let's say we have a total of 16 GPUs denoted by g0 ... g15 and we
69 + use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize
70 + the model pipeline. The present function will
71 + create 8 tensor model-parallel groups, 4 pipeline model-parallel groups
72 + and 8 data-parallel groups as:
73 + 8 data_parallel groups:
74 + [g0, g2], [g1, g3], [g4, g6], [g5, g7], [g8, g10], [g9, g11], [g12, g14], [g13, g15]
75 + 8 tensor model-parallel groups:
76 + [g0, g1], [g2, g3], [g4, g5], [g6, g7], [g8, g9], [g10, g11], [g12, g13], [g14, g15]
77 + 4 pipeline model-parallel groups:
78 + [g0, g4, g8, g12], [g1, g5, g9, g13], [g2, g6, g10, g14], [g3, g7, g11, g15]
79 + Note that for efficiency, the caller should make sure adjacent ranks
80 + are on the same DGX box. For example if we are using 2 DGX-1 boxes
81 + with a total of 16 GPUs, rank 0 to 7 belong to the first box and
82 + ranks 8 to 15 belong to the second box.
83 + """
84 + # Get world size and rank. Ensure some consistencies.
85 + assert torch.distributed.is_initialized()
86 + world_size: int = torch.distributed.get_world_size()
87 +
88 + if world_size % (tensor_model_parallel_size * pipeline_model_parallel_size) != 0:
89 + raise RuntimeError(
90 + f"world_size ({world_size}) is not divisible by tensor_model_parallel_size "
91 + f"({tensor_model_parallel_size}) x pipeline_model_parallel_size ({pipeline_model_parallel_size})"
92 + )
93 +
94 + data_parallel_size: int = world_size // (tensor_model_parallel_size *
95 + pipeline_model_parallel_size)
96 +
97 + num_tensor_model_parallel_groups: int = world_size // tensor_model_parallel_size
98 + num_pipeline_model_parallel_groups: int = world_size // pipeline_model_parallel_size
99 + num_data_parallel_groups: int = world_size // data_parallel_size
100 +
101 + if virtual_pipeline_model_parallel_size is not None:
102 + if not pipeline_model_parallel_size > 2:
103 + raise RuntimeError("pipeline-model-parallel size should be greater than 2 with "
104 + "interleaved schedule")
105 + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
106 + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
107 + _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = 0
108 + _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = virtual_pipeline_model_parallel_size
109 +
110 + if pipeline_model_parallel_split_rank is not None:
111 + global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK
112 + _PIPELINE_MODEL_PARALLEL_SPLIT_RANK = pipeline_model_parallel_split_rank
113 +
114 + rank = torch.distributed.get_rank()
115 +
116 + # Build the data-parallel groups.
117 + global _DATA_PARALLEL_GROUP
118 + global _DATA_PARALLEL_GLOBAL_RANKS
119 + assert _DATA_PARALLEL_GROUP is None, 'data parallel group is already initialized'
120 + all_data_parallel_group_ranks = []
121 + for i in range(pipeline_model_parallel_size):
122 + start_rank = i * num_pipeline_model_parallel_groups
123 + end_rank = (i + 1) * num_pipeline_model_parallel_groups
124 + for j in range(tensor_model_parallel_size):
125 + ranks = range(start_rank + j, end_rank, tensor_model_parallel_size)
126 + all_data_parallel_group_ranks.append(list(ranks))
127 + group = torch.distributed.new_group(ranks)
128 + if rank in ranks:
129 + _DATA_PARALLEL_GROUP = group
130 + _DATA_PARALLEL_GLOBAL_RANKS = ranks
131 +
132 + # Build the model-parallel groups.
133 + global _MODEL_PARALLEL_GROUP
134 + assert _MODEL_PARALLEL_GROUP is None, 'model parallel group is already initialized'
135 + for i in range(data_parallel_size):
136 + ranks = [data_parallel_group_ranks[i]
137 + for data_parallel_group_ranks in all_data_parallel_group_ranks]
138 + group = torch.distributed.new_group(ranks)
139 + if rank in ranks:
140 + _MODEL_PARALLEL_GROUP = group
141 +
142 + # Build the tensor model-parallel groups.
143 + global _TENSOR_MODEL_PARALLEL_GROUP
144 + assert _TENSOR_MODEL_PARALLEL_GROUP is None, \
145 + 'tensor model parallel group is already initialized'
146 + for i in range(num_tensor_model_parallel_groups):
147 + ranks = range(i * tensor_model_parallel_size,
148 + (i + 1) * tensor_model_parallel_size)
149 + group = torch.distributed.new_group(ranks)
150 + if rank in ranks:
151 + _TENSOR_MODEL_PARALLEL_GROUP = group
152 +
153 + # Build the pipeline model-parallel groups and embedding groups
154 + # (first and last rank in each pipeline model-parallel group).
155 + global _PIPELINE_MODEL_PARALLEL_GROUP
156 + global _PIPELINE_GLOBAL_RANKS
157 + assert _PIPELINE_MODEL_PARALLEL_GROUP is None, \
158 + 'pipeline model parallel group is already initialized'
159 + global _EMBEDDING_GROUP
160 + global _EMBEDDING_GLOBAL_RANKS
161 + assert _EMBEDDING_GROUP is None, 'embedding group is already initialized'
162 + global _POSITION_EMBEDDING_GROUP
163 + global _POSITION_EMBEDDING_GLOBAL_RANKS
164 + assert _POSITION_EMBEDDING_GROUP is None, \
165 + 'position embedding group is already initialized'
166 + for i in range(num_pipeline_model_parallel_groups):
167 + ranks = range(i, world_size, num_pipeline_model_parallel_groups)
168 + group = torch.distributed.new_group(ranks)
169 + if rank in ranks:
170 + _PIPELINE_MODEL_PARALLEL_GROUP = group
171 + _PIPELINE_GLOBAL_RANKS = ranks
172 + # Setup embedding group (to exchange gradients between
173 + # first and last stages).
174 + if len(ranks) > 1:
175 + embedding_ranks = [ranks[0], ranks[-1]]
176 + position_embedding_ranks = [ranks[0]]
177 + if pipeline_model_parallel_split_rank is not None:
178 + if ranks[pipeline_model_parallel_split_rank] not in embedding_ranks:
179 + embedding_ranks = [ranks[0],
180 + ranks[pipeline_model_parallel_split_rank],
181 + ranks[-1]]
182 + if ranks[pipeline_model_parallel_split_rank] not in position_embedding_ranks:
183 + position_embedding_ranks = [ranks[0],
184 + ranks[pipeline_model_parallel_split_rank]]
185 + else:
186 + embedding_ranks = ranks
187 + position_embedding_ranks = ranks
188 +
189 + group = torch.distributed.new_group(embedding_ranks)
190 + if rank in embedding_ranks:
191 + _EMBEDDING_GROUP = group
192 + if rank in ranks:
193 + _EMBEDDING_GLOBAL_RANKS = embedding_ranks
194 +
195 + group = torch.distributed.new_group(position_embedding_ranks)
196 + if rank in position_embedding_ranks:
197 + _POSITION_EMBEDDING_GROUP = group
198 + if rank in ranks:
199 + _POSITION_EMBEDDING_GLOBAL_RANKS = position_embedding_ranks
200 +
201 + # Initialize global memory buffer
202 + # This isn't really "parallel state" but there isn't another good place to
203 + # put this. If we end up with a more generic initialization of megatron-core
204 + # we could stick it there
205 + _set_global_memory_buffer()
206 +
207 +
208 +def model_parallel_is_initialized():
209 + """Check if model and data parallel groups are initialized."""
210 + if _TENSOR_MODEL_PARALLEL_GROUP is None or \
211 + _PIPELINE_MODEL_PARALLEL_GROUP is None or \
212 + _DATA_PARALLEL_GROUP is None:
213 + return False
214 + return True
215 +
216 +
217 +def get_model_parallel_group():
218 + """Get the model parallel group the caller rank belongs to."""
219 + assert _MODEL_PARALLEL_GROUP is not None, \
220 + 'model parallel group is not initialized'
221 + return _MODEL_PARALLEL_GROUP
222 +
223 +
224 +def get_tensor_model_parallel_group():
225 + """Get the tensor model parallel group the caller rank belongs to."""
226 + assert _TENSOR_MODEL_PARALLEL_GROUP is not None, \
227 + 'intra_layer_model parallel group is not initialized'
228 + return _TENSOR_MODEL_PARALLEL_GROUP
229 +
230 +
231 +def get_pipeline_model_parallel_group():
232 + """Get the pipeline model parallel group the caller rank belongs to."""
233 + assert _PIPELINE_MODEL_PARALLEL_GROUP is not None, \
234 + 'pipeline_model parallel group is not initialized'
235 + return _PIPELINE_MODEL_PARALLEL_GROUP
236 +
237 +
238 +def get_data_parallel_group():
239 + """Get the data parallel group the caller rank belongs to."""
240 + assert _DATA_PARALLEL_GROUP is not None, \
241 + 'data parallel group is not initialized'
242 + return _DATA_PARALLEL_GROUP
243 +
244 +
245 +def get_embedding_group():
246 + """Get the embedding group the caller rank belongs to."""
247 + assert _EMBEDDING_GROUP is not None, \
248 + 'embedding group is not initialized'
249 + return _EMBEDDING_GROUP
250 +
251 +
252 +def get_position_embedding_group():
253 + """Get the position embedding group the caller rank belongs to."""
254 + assert _POSITION_EMBEDDING_GROUP is not None, \
255 + 'position embedding group is not initialized'
256 + return _POSITION_EMBEDDING_GROUP
257 +
258 +
259 +def set_tensor_model_parallel_world_size(world_size):
260 + """Set the tensor model parallel size"""
261 + global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
262 + _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = world_size
263 +
264 +
265 +def set_pipeline_model_parallel_world_size(world_size):
266 + """Set the pipeline model parallel size"""
267 + global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
268 + _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = world_size
269 +
270 +
271 +def get_tensor_model_parallel_world_size():
272 + """Return world size for the tensor model parallel group."""
273 + global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
274 + if _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE is not None:
275 + return _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
276 + return torch.distributed.get_world_size(group=get_tensor_model_parallel_group())
277 +
278 +
279 +def get_pipeline_model_parallel_world_size():
280 + """Return world size for the pipeline model parallel group."""
281 + global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
282 + if _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE is not None:
283 + return _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
284 + return torch.distributed.get_world_size(group=get_pipeline_model_parallel_group())
285 +
286 +
287 +def set_tensor_model_parallel_rank(rank):
288 + """Set tensor model parallel rank."""
289 + global _MPU_TENSOR_MODEL_PARALLEL_RANK
290 + _MPU_TENSOR_MODEL_PARALLEL_RANK = rank
291 +
292 +
293 +def set_pipeline_model_parallel_rank(rank):
294 + """Set pipeline model parallel rank."""
295 + global _MPU_PIPELINE_MODEL_PARALLEL_RANK
296 + _MPU_PIPELINE_MODEL_PARALLEL_RANK = rank
297 +
298 +
299 +def set_pipeline_model_parallel_split_rank(rank):
300 + """Set pipeline model parallel split rank."""
301 + global _MPU_PIPELINE_MODEL_PARALLEL_SPLIT_RANK
302 + _MPU_PIPELINE_MODEL_PARALLEL_SPLIT_RANK = rank
303 +
304 +
305 +def get_tensor_model_parallel_rank():
306 + """Return my rank for the tensor model parallel group."""
307 + global _MPU_TENSOR_MODEL_PARALLEL_RANK
308 + if _MPU_TENSOR_MODEL_PARALLEL_RANK is not None:
309 + return _MPU_TENSOR_MODEL_PARALLEL_RANK
310 + return torch.distributed.get_rank(group=get_tensor_model_parallel_group())
311 +
312 +
313 +def get_pipeline_model_parallel_rank():
314 + """Return my rank for the pipeline model parallel group."""
315 + global _MPU_PIPELINE_MODEL_PARALLEL_RANK
316 + if _MPU_PIPELINE_MODEL_PARALLEL_RANK is not None:
317 + return _MPU_PIPELINE_MODEL_PARALLEL_RANK
318 + return torch.distributed.get_rank(group=get_pipeline_model_parallel_group())
319 +
320 +
321 +
322 +def is_pipeline_first_stage(ignore_virtual=False):
323 + """Return True if in the first pipeline model-parallel stage, False otherwise."""
324 + if not ignore_virtual:
325 + if get_virtual_pipeline_model_parallel_world_size() is not None and \
326 + get_virtual_pipeline_model_parallel_rank() != 0:
327 + return False
328 + return get_pipeline_model_parallel_rank() == 0
329 +
330 +
331 +def is_pipeline_last_stage(ignore_virtual=False):
332 + """Return True if in the last pipeline model-parallel stage, False otherwise."""
333 + if not ignore_virtual:
334 + virtual_pipeline_model_parallel_world_size = \
335 + get_virtual_pipeline_model_parallel_world_size()
336 + if virtual_pipeline_model_parallel_world_size is not None and \
337 + get_virtual_pipeline_model_parallel_rank() != (
338 + virtual_pipeline_model_parallel_world_size - 1):
339 + return False
340 + return get_pipeline_model_parallel_rank() == (
341 + get_pipeline_model_parallel_world_size() - 1)
342 +
343 +
344 +def is_rank_in_embedding_group(ignore_virtual=False):
345 + """Return true if current rank is in embedding group, False otherwise."""
346 + rank = torch.distributed.get_rank()
347 + global _EMBEDDING_GLOBAL_RANKS
348 + if ignore_virtual:
349 + return rank in _EMBEDDING_GLOBAL_RANKS
350 + if rank in _EMBEDDING_GLOBAL_RANKS:
351 + if rank == _EMBEDDING_GLOBAL_RANKS[0]:
352 + return is_pipeline_first_stage(ignore_virtual=False)
353 + elif rank == _EMBEDDING_GLOBAL_RANKS[-1]:
354 + return is_pipeline_last_stage(ignore_virtual=False)
355 + else:
356 + return True
357 + return False
358 +
359 +
360 +def is_rank_in_position_embedding_group():
361 + """Return true if current rank is in position embedding group, False otherwise."""
362 + rank = torch.distributed.get_rank()
363 + global _POSITION_EMBEDDING_GLOBAL_RANKS
364 + return rank in _POSITION_EMBEDDING_GLOBAL_RANKS
365 +
366 +
367 +def is_pipeline_stage_before_split(rank=None):
368 + """Return True if pipeline stage executes encoder block for a model
369 + with both encoder and decoder."""
370 + if get_pipeline_model_parallel_world_size() == 1:
371 + return True
372 + if rank is None:
373 + rank = get_pipeline_model_parallel_rank()
374 + global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK
375 + if _PIPELINE_MODEL_PARALLEL_SPLIT_RANK is None:
376 + return True
377 + if rank < _PIPELINE_MODEL_PARALLEL_SPLIT_RANK:
378 + return True
379 + return False
380 +
381 +
382 +def is_pipeline_stage_after_split(rank=None):
383 + """Return True if pipeline stage executes decoder block for a model
384 + with both encoder and decoder."""
385 + if get_pipeline_model_parallel_world_size() == 1:
386 + return True
387 + if rank is None:
388 + rank = get_pipeline_model_parallel_rank()
389 + global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK
390 + if _PIPELINE_MODEL_PARALLEL_SPLIT_RANK is None:
391 + return True
392 + if rank >= _PIPELINE_MODEL_PARALLEL_SPLIT_RANK:
393 + return True
394 + return False
395 +
396 +
397 +def is_pipeline_stage_at_split():
398 + """Return true if pipeline stage executes decoder block and next
399 + stage executes encoder block for a model with both encoder and
400 + decoder."""
401 + rank = get_pipeline_model_parallel_rank()
402 + return is_pipeline_stage_before_split(rank) and \
403 + is_pipeline_stage_after_split(rank+1)
404 +
405 +
406 +def get_virtual_pipeline_model_parallel_rank():
407 + """Return the virtual pipeline-parallel rank."""
408 + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
409 + return _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
410 +
411 +
412 +def set_virtual_pipeline_model_parallel_rank(rank):
413 + """Set the virtual pipeline-parallel rank."""
414 + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
415 + _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = rank
416 +
417 +
418 +def get_virtual_pipeline_model_parallel_world_size():
419 + """Return the virtual pipeline-parallel world size."""
420 + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
421 + return _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
422 +
423 +
424 +def get_tensor_model_parallel_src_rank():
425 + """Calculate the global rank corresponding to the first local rank
426 + in the tensor model parallel group."""
427 + global_rank = torch.distributed.get_rank()
428 + local_world_size = get_tensor_model_parallel_world_size()
429 + return (global_rank // local_world_size) * local_world_size
430 +
431 +
432 +def get_data_parallel_src_rank():
433 + """Calculate the global rank corresponding to the first local rank
434 + in the data parallel group."""
435 + assert _DATA_PARALLEL_GLOBAL_RANKS is not None, \
436 + "Data parallel group is not initialized"
437 + return _DATA_PARALLEL_GLOBAL_RANKS[0]
438 +
439 +
440 +def get_pipeline_model_parallel_first_rank():
441 + """Return the global rank of the first process in the pipeline for the
442 + current tensor parallel group"""
443 + assert _PIPELINE_GLOBAL_RANKS is not None, \
444 + "Pipeline parallel group is not initialized"
445 + return _PIPELINE_GLOBAL_RANKS[0]
446 +
447 +
448 +def get_pipeline_model_parallel_last_rank():
449 + """Return the global rank of the last process in the pipeline for the
450 + current tensor parallel group"""
451 + assert _PIPELINE_GLOBAL_RANKS is not None, \
452 + "Pipeline parallel group is not initialized"
453 + last_rank_local = get_pipeline_model_parallel_world_size() - 1
454 + return _PIPELINE_GLOBAL_RANKS[last_rank_local]
455 +
456 +def get_pipeline_model_parallel_next_rank():
457 + """Return the global rank that follows the caller in the pipeline"""
458 + assert _PIPELINE_GLOBAL_RANKS is not None, \
459 + "Pipeline parallel group is not initialized"
460 + rank_in_pipeline = get_pipeline_model_parallel_rank()
461 + world_size = get_pipeline_model_parallel_world_size()
462 + return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline + 1) % world_size]
463 +
464 +
465 +def get_pipeline_model_parallel_prev_rank():
466 + """Return the global rank that preceeds the caller in the pipeline"""
467 + assert _PIPELINE_GLOBAL_RANKS is not None, \
468 + "Pipeline parallel group is not initialized"
469 + rank_in_pipeline = get_pipeline_model_parallel_rank()
470 + world_size = get_pipeline_model_parallel_world_size()
471 + return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline - 1) % world_size]
472 +
473 +
474 +def get_data_parallel_world_size():
475 + """Return world size for the data parallel group."""
476 + return torch.distributed.get_world_size(group=get_data_parallel_group())
477 +
478 +
479 +def get_data_parallel_rank():
480 + """Return my rank for the data parallel group."""
481 + return torch.distributed.get_rank(group=get_data_parallel_group())
482 +
483 +def _set_global_memory_buffer():
484 + """Initialize global buffer"""
485 + global _GLOBAL_MEMORY_BUFFER
486 + assert _GLOBAL_MEMORY_BUFFER is None, 'global memory buffer is already initialized'
487 + _GLOBAL_MEMORY_BUFFER = GlobalMemoryBuffer()
488 +
489 +def get_global_memory_buffer():
490 + """Return the global GlobalMemoryBuffer object"""
491 + assert _GLOBAL_MEMORY_BUFFER is not None, 'global memory buffer is not initialized'
492 + return _GLOBAL_MEMORY_BUFFER
493 +
494 +
495 +def destroy_model_parallel():
496 + """Set the groups to none."""
497 + global _MODEL_PARALLEL_GROUP
498 + _MODEL_PARALLEL_GROUP = None
499 + global _TENSOR_MODEL_PARALLEL_GROUP
500 + _TENSOR_MODEL_PARALLEL_GROUP = None
501 + global _PIPELINE_MODEL_PARALLEL_GROUP
502 + _PIPELINE_MODEL_PARALLEL_GROUP = None
503 + global _DATA_PARALLEL_GROUP
504 + _DATA_PARALLEL_GROUP = None
505 + global _EMBEDDING_GROUP
506 + _EMBEDDING_GROUP = None
507 + global _POSITION_EMBEDDING_GROUP
508 + _POSITION_EMBEDDING_GROUP = None
509 + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
510 + _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = None
511 + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
512 + _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
513 + global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
514 + _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = None
515 + global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
516 + _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
517 + global _MPU_TENSOR_MODEL_PARALLEL_RANK
518 + _MPU_TENSOR_MODEL_PARALLEL_RANK = None
519 + global _MPU_PIPELINE_MODEL_PARALLEL_RANK
520 + _MPU_PIPELINE_MODEL_PARALLEL_RANK = None
521 + global _GLOBAL_MEMORY_BUFFER
522 + _GLOBAL_MEMORY_BUFFER = None
cacheflow/parallel_utils/tensor_parallel/__init__.py new
+58
@@ -0,0 +1,58 @@
1 +from .layers import (
2 + ColumnParallelLinear,
3 + RowParallelLinear,
4 + VocabParallelEmbedding,
5 + set_tensor_model_parallel_attributes,
6 + set_defaults_if_not_set_tensor_model_parallel_attributes,
7 + copy_tensor_model_parallel_attributes,
8 + param_is_not_tensor_parallel_duplicate,
9 + linear_with_grad_accumulation_and_async_allreduce
10 +
11 +)
12 +
13 +from .mappings import (
14 + copy_to_tensor_model_parallel_region,
15 + gather_from_tensor_model_parallel_region,
16 + gather_from_sequence_parallel_region,
17 + scatter_to_tensor_model_parallel_region,
18 + scatter_to_sequence_parallel_region,
19 +)
20 +
21 +from .random import (
22 + checkpoint,
23 + get_cuda_rng_tracker,
24 + model_parallel_cuda_manual_seed,
25 +)
26 +
27 +from .utils import (
28 + split_tensor_along_last_dim,
29 + split_tensor_into_1d_equal_chunks,
30 + gather_split_1d_tensor,
31 +)
32 +
33 +__all__ = [
34 + #layers.py
35 + "ColumnParallelLinear",
36 + "RowParallelLinear",
37 + "VocabParallelEmbedding",
38 + "set_tensor_model_parallel_attributes",
39 + "set_defaults_if_not_set_tensor_model_parallel_attributes",
40 + "copy_tensor_model_parallel_attributes",
41 + "param_is_not_tensor_parallel_duplicate",
42 + "linear_with_grad_accumulation_and_async_allreduce",
43 + # mappings.py
44 + "copy_to_tensor_model_parallel_region",
45 + "gather_from_tensor_model_parallel_region",
46 + "gather_from_sequence_parallel_region",
47 +# "reduce_from_tensor_model_parallel_region",
48 + "scatter_to_tensor_model_parallel_region",
49 + "scatter_to_sequence_parallel_region",
50 + # random.py
51 + "checkpoint",
52 + "get_cuda_rng_tracker",
53 + "model_parallel_cuda_manual_seed",
54 + # utils.py
55 + "split_tensor_along_last_dim",
56 + "split_tensor_into_1d_equal_chunks",
57 + "gather_split_1d_tensor",
58 +]
cacheflow/parallel_utils/tensor_parallel/layers.py new
+719
@@ -0,0 +1,719 @@
1 +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2 +
3 +# Parts of the code here are adapted from PyTorch
4 +# repo: https://github.com/pytorch/pytorch
5 +
6 +import math
7 +import os
8 +from typing import Optional
9 +import warnings
10 +
11 +import torch
12 +import torch.nn.functional as F
13 +import torch.nn.init as init
14 +from torch.nn.parameter import Parameter
15 +
16 +from cacheflow.parallel_utils.parallel_state import (
17 + get_tensor_model_parallel_rank,
18 + get_tensor_model_parallel_world_size,
19 + get_tensor_model_parallel_group,
20 + get_global_memory_buffer,
21 +)
22 +from .mappings import (
23 + copy_to_tensor_model_parallel_region,
24 + gather_from_tensor_model_parallel_region,
25 + gather_from_sequence_parallel_region,
26 + reduce_from_tensor_model_parallel_region,
27 + scatter_to_tensor_model_parallel_region,
28 + reduce_scatter_to_sequence_parallel_region,
29 +)
30 +
31 +from .random import get_cuda_rng_tracker
32 +from .utils import (
33 + divide,
34 + split_tensor_along_last_dim,
35 + VocabUtility,
36 +)
37 +
38 +_grad_accum_fusion_available = True
39 +try:
40 + import fused_weight_gradient_mlp_cuda
41 +except ImportError:
42 + _grad_accum_fusion_available = False
43 +
44 +_MODEL_PARALLEL_ATTRIBUTE_DEFAULTS = {'tensor_model_parallel': False,
45 + 'partition_dim': -1,
46 + 'partition_stride': 1}
47 +
48 +def param_is_not_tensor_parallel_duplicate(param):
49 + return (hasattr(param, 'tensor_model_parallel') and
50 + param.tensor_model_parallel) or (
51 + get_tensor_model_parallel_rank() == 0)
52 +
53 +
54 +def set_tensor_model_parallel_attributes(tensor, is_parallel, dim, stride):
55 + # Make sure the attributes are not set.
56 + for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS:
57 + assert not hasattr(tensor, attribute)
58 + # Set the attributes.
59 + setattr(tensor, 'tensor_model_parallel', is_parallel)
60 + setattr(tensor, 'partition_dim', dim)
61 + setattr(tensor, 'partition_stride', stride)
62 +
63 +
64 +def set_defaults_if_not_set_tensor_model_parallel_attributes(tensor):
65 + def maybe_set(attribute, value):
66 + if not hasattr(tensor, attribute):
67 + setattr(tensor, attribute, value)
68 + for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS:
69 + maybe_set(attribute, _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS[attribute])
70 +
71 +
72 +def copy_tensor_model_parallel_attributes(destination_tensor, source_tensor):
73 + def maybe_copy(attribute):
74 + if hasattr(source_tensor, attribute):
75 + setattr(destination_tensor, attribute,
76 + getattr(source_tensor, attribute))
77 + for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS:
78 + maybe_copy(attribute)
79 +
80 +
81 +def _initialize_affine_weight_gpu(weight, init_method,
82 + partition_dim, stride=1):
83 + """Initialize affine weight for model parallel on GPU."""
84 +
85 + set_tensor_model_parallel_attributes(tensor=weight,
86 + is_parallel=True,
87 + dim=partition_dim,
88 + stride=stride)
89 +
90 + with get_cuda_rng_tracker().fork():
91 + init_method(weight)
92 +
93 +
94 +def _initialize_affine_weight_cpu(weight, output_size, input_size,
95 + per_partition_size, partition_dim,
96 + init_method, stride=1,
97 + return_master_weight=False,
98 + *, params_dtype=None):
99 + """Initialize affine weight for model parallel.
100 +
101 + Build the master weight on all processes and scatter
102 + the relevant chunk."""
103 +
104 + set_tensor_model_parallel_attributes(tensor=weight,
105 + is_parallel=True,
106 + dim=partition_dim,
107 + stride=stride)
108 +
109 + if params_dtype is None:
110 + params_dtype = torch.get_default_dtype()
111 +
112 + # Initialize master weight
113 + master_weight = torch.empty(output_size, input_size,
114 + dtype=torch.float,
115 + requires_grad=False)
116 + init_method(master_weight)
117 + master_weight = master_weight.to(dtype=params_dtype)
118 +
119 + # Split and copy
120 + per_partition_per_stride_size = divide(per_partition_size, stride)
121 + weight_list = torch.split(master_weight, per_partition_per_stride_size,
122 + dim=partition_dim)
123 + rank = get_tensor_model_parallel_rank()
124 + world_size = get_tensor_model_parallel_world_size()
125 + my_weight_list = weight_list[rank::world_size]
126 +
127 + with torch.no_grad():
128 + torch.cat(my_weight_list, dim=partition_dim, out=weight)
129 + if return_master_weight:
130 + return master_weight
131 + return None
132 +
133 +
134 +class VocabParallelEmbedding(torch.nn.Module):
135 + """Embedding parallelized in the vocabulary dimension.
136 +
137 + This is mainly adapted from torch.nn.Embedding and all the default
138 + values are kept.
139 + Arguments:
140 + num_embeddings: vocabulary size.
141 + embedding_dim: size of hidden state.
142 +
143 + Keyword Arguments:
144 + init_method: method to initialize weights.
145 + params_dtype
146 + use_cpu_initialization
147 + perform_initialization
148 + """
149 +
150 + def __init__(self, num_embeddings: int, embedding_dim: int, *,
151 + init_method=init.xavier_normal_,
152 + params_dtype: torch.dtype=None,
153 + use_cpu_initialization: bool=False,
154 + perform_initialization: bool=True):
155 + super(VocabParallelEmbedding, self).__init__()
156 + # Keep the input dimensions.
157 + self.num_embeddings = num_embeddings
158 + self.embedding_dim = embedding_dim
159 + if params_dtype is None:
160 + params_dtype = torch.get_default_dtype()
161 +
162 + # Set the defaults for compatibility.
163 + self.padding_idx = None
164 + self.max_norm = None
165 + self.norm_type = 2.
166 + self.scale_grad_by_freq = False
167 + self.sparse = False
168 + self._weight = None
169 + self.tensor_model_parallel_size = get_tensor_model_parallel_world_size()
170 + # Divide the weight matrix along the vocaburaly dimension.
171 + self.vocab_start_index, self.vocab_end_index = \
172 + VocabUtility.vocab_range_from_global_vocab_size(
173 + self.num_embeddings, get_tensor_model_parallel_rank(),
174 + self.tensor_model_parallel_size)
175 + self.num_embeddings_per_partition = self.vocab_end_index - \
176 + self.vocab_start_index
177 +
178 + # Allocate weights and initialize.
179 + if use_cpu_initialization:
180 + self.weight = Parameter(torch.empty(
181 + self.num_embeddings_per_partition, self.embedding_dim,
182 + dtype=params_dtype))
183 + if perform_initialization:
184 + _initialize_affine_weight_cpu(
185 + self.weight, self.num_embeddings, self.embedding_dim,
186 + self.num_embeddings_per_partition, 0, init_method,
187 + params_dtype=params_dtype)
188 + else:
189 + self.weight = Parameter(torch.empty(
190 + self.num_embeddings_per_partition, self.embedding_dim,
191 + device=torch.cuda.current_device(), dtype=params_dtype))
192 + if perform_initialization:
193 + _initialize_affine_weight_gpu(self.weight, init_method,
194 + partition_dim=0, stride=1)
195 +
196 + def forward(self, input_):
197 + if self.tensor_model_parallel_size > 1:
198 + # Build the mask.
199 + input_mask = (input_ < self.vocab_start_index) | \
200 + (input_ >= self.vocab_end_index)
201 + # Mask the input.
202 + masked_input = input_.clone() - self.vocab_start_index
203 + masked_input[input_mask] = 0
204 + else:
205 + masked_input = input_
206 + # Get the embeddings.
207 + output_parallel = F.embedding(masked_input, self.weight,
208 + self.padding_idx, self.max_norm,
209 + self.norm_type, self.scale_grad_by_freq,
210 + self.sparse)
211 + # Mask the output embedding.
212 + if self.tensor_model_parallel_size > 1:
213 + output_parallel[input_mask, :] = 0.0
214 + # Reduce across all the model parallel GPUs.
215 + output = reduce_from_tensor_model_parallel_region(output_parallel)
216 + return output
217 +
218 +
219 +class LinearWithGradAccumulationAndAsyncCommunication(torch.autograd.Function):
220 + """See linear_with_grad_accumulation_and_async_allreduce"""
221 +
222 + @staticmethod
223 + def forward(ctx, input, weight, bias, gradient_accumulation_fusion,
224 + async_grad_allreduce, sequence_parallel):
225 + ctx.save_for_backward(input, weight)
226 + ctx.use_bias = bias is not None
227 + ctx.gradient_accumulation_fusion = gradient_accumulation_fusion
228 + ctx.async_grad_allreduce = async_grad_allreduce
229 + ctx.sequence_parallel = sequence_parallel
230 +
231 + if sequence_parallel:
232 + world_size = get_tensor_model_parallel_world_size()
233 + dim_size = list(input.size())
234 + dim_size[0] = dim_size[0] * world_size
235 +
236 + all_gather_buffer = \
237 + get_global_memory_buffer().get_tensor(dim_size, input.dtype, "mpu")
238 + torch.distributed._all_gather_base(
239 + all_gather_buffer,
240 + input,
241 + group=get_tensor_model_parallel_group())
242 + total_input = all_gather_buffer
243 + else:
244 + total_input = input
245 +
246 + output = torch.matmul(total_input, weight.t())
247 + if bias is not None:
248 + output = output + bias
249 + return output
250 +
251 + @staticmethod
252 + def backward(ctx, grad_output):
253 + input, weight = ctx.saved_tensors
254 + use_bias = ctx.use_bias
255 +
256 + if ctx.sequence_parallel:
257 + world_size = get_tensor_model_parallel_world_size()
258 + dim_size = list(input.size())
259 + dim_size[0] = dim_size[0] * world_size
260 +
261 + all_gather_buffer = \
262 + get_global_memory_buffer().get_tensor(dim_size, input.dtype, "mpu")
263 + handle = torch.distributed._all_gather_base(
264 + all_gather_buffer,
265 + input,
266 + group=get_tensor_model_parallel_group(), async_op=True)
267 +
268 + # Here we rely on CUDA_DEVICE_MAX_CONNECTIONS=1 to ensure that the
269 + # gather is scheduled before the input gradient computation
270 + total_input = all_gather_buffer
271 + else:
272 + total_input = input
273 + grad_input = grad_output.matmul(weight)
274 +
275 + if ctx.sequence_parallel:
276 + handle.wait()
277 +
278 + # Convert the tensor shapes to 2D for execution compatibility
279 + grad_output = grad_output.view(grad_output.shape[0] * grad_output.shape[1],
280 + grad_output.shape[2])
281 + total_input = total_input.view(total_input.shape[0] * total_input.shape[1],
282 + total_input.shape[2])
283 +
284 + if ctx.async_grad_allreduce:
285 + # Asynchronous all-reduce
286 + handle = torch.distributed.all_reduce(
287 + grad_input, group=get_tensor_model_parallel_group(), async_op=True)
288 + # Here we rely on CUDA_DEVICE_MAX_CONNECTIONS=1 to ensure that the
289 + # all-reduce is scheduled before the weight gradient computation
290 +
291 + if ctx.sequence_parallel:
292 + assert not ctx.async_grad_allreduce
293 + dim_size = list(input.size())
294 + sub_grad_input = torch.empty(dim_size, dtype=input.dtype,
295 + device=torch.cuda.current_device(),
296 + requires_grad=False)
297 + # reduce_scatter
298 + handle = torch.distributed._reduce_scatter_base(sub_grad_input, grad_input,
299 + group=get_tensor_model_parallel_group(),
300 + async_op=True)
301 + # Here we rely on CUDA_DEVICE_MAX_CONNECTIONS=1 to ensure that the
302 + # reduce scatter is scheduled before the weight gradient computation
303 +
304 +
305 + if ctx.gradient_accumulation_fusion:
306 + if weight.main_grad.dtype == torch.float32:
307 + fused_weight_gradient_mlp_cuda.wgrad_gemm_accum_fp32(total_input, grad_output, weight.main_grad)
308 + elif weight.main_grad.dtype == torch.float16:
309 + fused_weight_gradient_mlp_cuda.wgrad_gemm_accum_fp16(total_input, grad_output, weight.main_grad)
310 + else:
311 + raise RuntimeError("Unsupported gradient type for gradient accumulation fusion")
312 + grad_weight = None
313 + else:
314 + grad_weight = grad_output.t().matmul(total_input)
315 + grad_bias = grad_output.sum(dim=0) if use_bias else None
316 +
317 + if ctx.sequence_parallel:
318 + handle.wait()
319 + return sub_grad_input, grad_weight, grad_bias, None, None, None
320 +
321 + if ctx.async_grad_allreduce:
322 + handle.wait()
323 +
324 + return grad_input, grad_weight, grad_bias, None, None, None
325 +
326 +def linear_with_grad_accumulation_and_async_allreduce(
327 + input: torch.Tensor,
328 + weight: torch.Tensor,
329 + bias: Optional[torch.Tensor],
330 + gradient_accumulation_fusion: bool,
331 + async_grad_allreduce: bool,
332 + sequence_parallel_enabled: bool,
333 +) -> torch.Tensor:
334 + """Linear layer execution with asynchronous communication and
335 + gradient accumulation fusion in backprop.
336 +
337 + This has the option to accumulate the result of backprop
338 + calculation into an existing gradient buffer, preventing the need
339 + to do an additional addition kernel after the gradient
340 + calculation.
341 +
342 + Additionally, the tensor parallel all reduce of the input
343 + gradients can be done asynchronously with the calculation of
344 + the weight gradients.
345 +
346 + In the case of sequence parallelism, the reduce scatter of the
347 + input gradients is done asynchronously with the calcluation of the
348 + weight gradients.
349 +
350 + Use of this module requires that the environment variable
351 + CUDA_DEVICE_MAX_CONNECTIONS=1. There are a few collective
352 + operations, noted in the code, that should be scheduled before
353 + compute kernels to overlap the communication with the computation,
354 + which is necessary for a speedup but not for correctness so that
355 + ordering isn't imposed by the scheduler. Setting
356 + CUDA_DEVICE_MAX_CONNECTIONS=1 forces the kernels to be scheduled
357 + in the order they are called.
358 +
359 + Arguments:
360 +
361 + input (torch.Tensor required): input like torch.nn.functional.linear
362 +
363 + weight (torch.Tensor required): weight like torch.nn.functional.linear
364 +
365 + bias (torch.Tensor optional): bias like torch.nn.functional.linear
366 +
367 + gradient_accumulation_fusion (bool required): Perform the gradient
368 + accumulation fusion, requires the custom CUDA extension
369 + fused_weight_gradient_mlp_cuda module. To use
370 + gradient_accumulation_fusion you must install APEX with
371 + --cpp_ext and --cuda_ext. For example: "pip install
372 + --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext .\"
373 + " Note that the extension requires CUDA>=11. Otherwise, you
374 + must turn off gradient accumulation fusion."
375 +
376 + async_grad_allreduce (bool required): Do the allreduce of input
377 + gradients asyncronously with the computation of weight
378 + gradients. If sequence_parallel_enabled is True, this must be
379 + False, as no all reduce is performed.
380 +
381 + sequence_parallel_enabled (bool required): Indicates that sequence
382 + parallelism is used and thus in the forward pass the input is
383 + all gathered, and the backward pass the input gradients are
384 + reduce scattered.
385 + """
386 + args = [
387 + input,
388 + weight,
389 + bias,
390 + gradient_accumulation_fusion,
391 + async_grad_allreduce,
392 + sequence_parallel_enabled,
393 + ]
394 +
395 + if not linear_with_grad_accumulation_and_async_allreduce.warned:
396 + if os.environ.get('CUDA_DEVICE_MAX_CONNECTIONS') != "1":
397 + if sequence_parallel_enabled:
398 + warnings.warn(
399 + "When using sequence parallelism it is recommended to set the "
400 + "environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 for "
401 + "maximum speedup")
402 + linear_with_grad_accumulation_and_async_allreduce.warned = True
403 +
404 + if async_grad_allreduce:
405 + warnings.warn(
406 + "When using async grad allreduce it is recommended to set the "
407 + "environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 for "
408 + "maximum speedup")
409 + linear_with_grad_accumulation_and_async_allreduce.warned = True
410 +
411 + with torch.cuda.amp.autocast(enabled=False):
412 + return LinearWithGradAccumulationAndAsyncCommunication.apply(*args)
413 +linear_with_grad_accumulation_and_async_allreduce.warned = False
414 +
415 +class ColumnParallelLinear(torch.nn.Module):
416 + """Linear layer with column parallelism.
417 +
418 + The linear layer is defined as Y = XA + b. A is parallelized along
419 + its second dimension as A = [A_1, ..., A_p].
420 +
421 + Arguments:
422 + input_size: first dimension of matrix A.
423 + output_size: second dimension of matrix A.
424 +
425 + Keyword Arguments
426 + bias: If true, add bias
427 + gather_output: If true, call all-gather on output and make Y available
428 + to all GPUs, otherwise, every GPU will have its output
429 + which is Y_i = XA_i
430 + init_method: method to initialize weights. Note that bias is always set
431 + to zero.
432 + stride: For the strided linear layers.
433 + keep_master_weight_for_test: This was added for testing and should be
434 + set to False. It returns the master weights
435 + used for initialization.
436 + skip_bias_add: This was added to enable performance optimations where bias
437 + can be fused with other elementwise operations. we skip
438 + adding bias but instead return it.
439 + async_tensor_model_parallel_allreduce:
440 + params_dtype:
441 + use_cpu_initialization:
442 + gradient_accumulation_fusion:
443 + sequence_parallel_enabled:
444 + """
445 +
446 + def __init__(self, input_size, output_size, *,
447 + bias=True, gather_output=True,
448 + init_method=init.xavier_normal_, stride=1,
449 + keep_master_weight_for_test=False,
450 + skip_bias_add=False,
451 + async_tensor_model_parallel_allreduce=True,
452 + params_dtype=None,
453 + use_cpu_initialization=False,
454 + perform_initialization=True,
455 + gradient_accumulation_fusion=False,
456 + sequence_parallel_enabled: bool = False,
457 + ):
458 + super(ColumnParallelLinear, self).__init__()
459 +
460 + # Keep input parameters
461 + self.input_size = input_size
462 + self.output_size = output_size
463 + self.gather_output = gather_output
464 + # Divide the weight matrix along the last dimension.
465 + world_size = get_tensor_model_parallel_world_size()
466 + self.output_size_per_partition = divide(output_size, world_size)
467 + self.skip_bias_add = skip_bias_add
468 +
469 + if params_dtype is None:
470 + params_dtype = torch.get_default_dtype()
471 +
472 + # Parameters.
473 + # Note: torch.nn.functional.linear performs XA^T + b and as a result
474 + # we allocate the transpose.
475 + # Initialize weight.
476 + if use_cpu_initialization:
477 + self.weight = Parameter(torch.empty(self.output_size_per_partition,
478 + self.input_size,
479 + dtype=params_dtype))
480 + if perform_initialization:
481 + self.master_weight = _initialize_affine_weight_cpu(
482 + self.weight, self.output_size, self.input_size,
483 + self.output_size_per_partition, 0, init_method,
484 + stride=stride, return_master_weight=keep_master_weight_for_test)
485 + else:
486 + self.weight = Parameter(torch.empty(
487 + self.output_size_per_partition, self.input_size,
488 + device=torch.cuda.current_device(), dtype=params_dtype))
489 + if perform_initialization:
490 + _initialize_affine_weight_gpu(self.weight, init_method,
491 + partition_dim=0, stride=stride)
492 +
493 + if bias:
494 + if use_cpu_initialization:
495 + self.bias = Parameter(torch.empty(
496 + self.output_size_per_partition, dtype=params_dtype))
497 + else:
498 + self.bias = Parameter(torch.empty(
499 + self.output_size_per_partition,
500 + device=torch.cuda.current_device(),
501 + dtype=params_dtype))
502 + set_tensor_model_parallel_attributes(self.bias, True, 0, stride)
503 + # Always initialize bias to zero.
504 + with torch.no_grad():
505 + self.bias.zero_()
506 + else:
507 + self.register_parameter('bias', None)
508 +
509 + self.async_tensor_model_parallel_allreduce = (
510 + async_tensor_model_parallel_allreduce and
511 + world_size > 1)
512 + if sequence_parallel_enabled:
513 + if world_size <= 1:
514 + warnings.warn(
515 + f"`sequence_parallel_enabled` is set to `True`, but tensor model parallel size is {world_size}. "
516 + f"Disabling sequence parallel."
517 + )
518 + sequence_parallel_enabled = False
519 + self.sequence_parallel_enabled = sequence_parallel_enabled
520 +
521 + if gradient_accumulation_fusion:
522 + if not _grad_accum_fusion_available:
523 + raise RuntimeError(
524 + "ColumnParallelLinear was called with gradient_accumulation_fusion set "
525 + "to True but the custom CUDA extension fused_weight_gradient_mlp_cuda "
526 + "module is not found. To use gradient_accumulation_fusion you must "
527 + "install APEX with --cpp_ext and --cuda_ext. For example: "
528 + "pip install --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext .\" "
529 + "Note that the extension requires CUDA>=11. Otherwise, you must turn off "
530 + "gradient accumulation fusion."
531 + )
532 + self.gradient_accumulation_fusion = gradient_accumulation_fusion
533 +
534 + if self.async_tensor_model_parallel_allreduce and self.sequence_parallel_enabled:
535 + raise RuntimeError(
536 + "`async_tensor_model_parallel_allreduce` and `sequence_parallel_enabled` "
537 + "cannot be enabled at the same time."
538 + )
539 +
540 +
541 + def forward(self, input_):
542 + """Forward of ColumnParallelLinear
543 +
544 + Args:
545 + input_: 3D tensor whose order of dimension is [sequence, batch, hidden]
546 +
547 + Returns:
548 + - output
549 + - bias
550 + """
551 + bias = self.bias if not self.skip_bias_add else None
552 +
553 + if self.async_tensor_model_parallel_allreduce or \
554 + self.sequence_parallel_enabled:
555 + input_parallel = input_
556 + else:
557 + input_parallel = copy_to_tensor_model_parallel_region(input_)
558 + # Matrix multiply.
559 + output_parallel = linear_with_grad_accumulation_and_async_allreduce(
560 + input=input_parallel,
561 + weight=self.weight,
562 + bias=bias,
563 + gradient_accumulation_fusion=self.gradient_accumulation_fusion,
564 + async_grad_allreduce=self.async_tensor_model_parallel_allreduce,
565 + sequence_parallel_enabled=self.sequence_parallel_enabled,
566 + )
567 + if self.gather_output:
568 + # All-gather across the partitions.
569 + assert not self.sequence_parallel_enabled
570 + output = gather_from_tensor_model_parallel_region(output_parallel)
571 + else:
572 + output = output_parallel
573 + output_bias = self.bias if self.skip_bias_add else None
574 + return output, output_bias
575 +
576 +
577 +class RowParallelLinear(torch.nn.Module):
578 + """Linear layer with row parallelism.
579 +
580 + The linear layer is defined as Y = XA + b. A is parallelized along
581 + its first dimension and X along its second dimension as:
582 + - -
583 + | A_1 |
584 + | . |
585 + A = | . | X = [X_1, ..., X_p]
586 + | . |
587 + | A_p |
588 + - -
589 + Arguments:
590 + input_size: first dimension of matrix A.
591 + output_size: second dimension of matrix A.
592 +
593 + Keyword Arguments:
594 + bias: If true, add bias. Note that bias is not parallelized.
595 + input_is_parallel: If true, we assume that the input is already
596 + split across the GPUs and we do not split
597 + again.
598 + init_method: method to initialize weights. Note that bias is always set
599 + to zero.
600 + stride: For the strided linear layers.
601 + keep_master_weight_for_test: This was added for testing and should be
602 + set to False. It returns the master weights
603 + used for initialization.
604 + skip_bias_add: This was added to enable performance optimization where bias
605 + can be fused with other elementwise operations. We skip
606 + adding bias but instead return it.
607 + params_dtype:
608 + use_cpu_initialization:
609 + perform_initialization:
610 + gradient_accumulation_fusion:
611 + sequence_parallel_enabled:
612 + """
613 +
614 + def __init__(self, input_size, output_size, *,
615 + bias=True, input_is_parallel=False,
616 + init_method=init.xavier_normal_, stride=1,
617 + keep_master_weight_for_test=False,
618 + skip_bias_add=False,
619 + params_dtype=None,
620 + use_cpu_initialization=False,
621 + perform_initialization=True,
622 + gradient_accumulation_fusion=False,
623 + sequence_parallel_enabled: bool = False,
624 + ):
625 + super(RowParallelLinear, self).__init__()
626 +
627 + # Keep input parameters
628 + self.input_size = input_size
629 + self.output_size = output_size
630 + self.input_is_parallel = input_is_parallel
631 + if params_dtype is None:
632 + params_dtype = torch.get_default_dtype()
633 +
634 + # Divide the weight matrix along the last dimension.
635 + world_size = get_tensor_model_parallel_world_size()
636 + self.input_size_per_partition = divide(input_size, world_size)
637 + self.skip_bias_add = skip_bias_add
638 + self.gradient_accumulation_fusion = gradient_accumulation_fusion
639 + self.sequence_parallel_enabled = sequence_parallel_enabled
640 + if self.sequence_parallel_enabled and not self.input_is_parallel:
641 + raise RuntimeError("To enable `sequence_parallel_enabled`, `input_is_parallel` must be `True`")
642 +
643 + # Parameters.
644 + # Note: torch.nn.functional.linear performs XA^T + b and as a result
645 + # we allocate the transpose.
646 + # Initialize weight.
647 + if use_cpu_initialization:
648 + self.weight = Parameter(torch.empty(self.output_size,
649 + self.input_size_per_partition,
650 + dtype=params_dtype))
651 + if perform_initialization:
652 + self.master_weight = _initialize_affine_weight_cpu(
653 + self.weight, self.output_size, self.input_size,
654 + self.input_size_per_partition, 1, init_method,
655 + stride=stride, return_master_weight=keep_master_weight_for_test,
656 + params_dtype=params_dtype)
657 + else:
658 + self.weight = Parameter(torch.empty(
659 + self.output_size, self.input_size_per_partition,
660 + device=torch.cuda.current_device(), dtype=params_dtype))
661 + if perform_initialization:
662 + _initialize_affine_weight_gpu(self.weight, init_method,
663 + partition_dim=1, stride=stride)
664 + if bias:
665 + if use_cpu_initialization:
666 + self.bias = Parameter(torch.empty(self.output_size,
667 + dtype=params_dtype))
668 + else:
669 + self.bias = Parameter(torch.empty(
670 + self.output_size, device=torch.cuda.current_device(),
671 + dtype=params_dtype))
672 + setattr(self.bias, 'sequence_parallel', sequence_parallel_enabled)
673 +
674 + # Always initialize bias to zero.
675 + with torch.no_grad():
676 + self.bias.zero_()
677 + else:
678 + self.register_parameter('bias', None)
679 +
680 +
681 +
682 + def forward(self, input_):
683 + """Forward of RowParallelLinear
684 +
685 + Args:
686 + input_: 3D tensor whose order of dimension is [sequence, batch, hidden]
687 +
688 + Returns:
689 + - output
690 + - bias
691 + """
692 + # Set up backprop all-reduce.
693 + if self.input_is_parallel:
694 + input_parallel = input_
695 + else:
696 + assert not self.sequence_parallel_enabled
697 + input_parallel = scatter_to_tensor_model_parallel_region(input_)
698 + # Matrix multiply.
699 + output_parallel = linear_with_grad_accumulation_and_async_allreduce(
700 + input=input_parallel,
701 + weight=self.weight,
702 + bias=None,
703 + gradient_accumulation_fusion=self.gradient_accumulation_fusion,
704 + async_grad_allreduce=False,
705 + sequence_parallel_enabled=False,
706 + )
707 +
708 + # All-reduce across all the partitions.
709 + if self.sequence_parallel_enabled:
710 + output_ = reduce_scatter_to_sequence_parallel_region(output_parallel)
711 + else:
712 + output_ = reduce_from_tensor_model_parallel_region(output_parallel)
713 + if not self.skip_bias_add:
714 + output = output_ + self.bias if self.bias is not None else output_
715 + output_bias = None
716 + else:
717 + output = output_
718 + output_bias = self.bias
719 + return output, output_bias
cacheflow/parallel_utils/tensor_parallel/mappings.py new
+279
@@ -0,0 +1,279 @@
1 +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2 +
3 +import torch
4 +
5 +from cacheflow.parallel_utils.parallel_state import (
6 + get_tensor_model_parallel_rank,
7 + get_tensor_model_parallel_world_size,
8 + get_tensor_model_parallel_group,
9 +)
10 +from .utils import split_tensor_along_last_dim
11 +
12 +
13 +def _reduce(input_):
14 + """All-reduce the input tensor across model parallel group."""
15 +
16 + # Bypass the function if we are using only 1 GPU.
17 + if get_tensor_model_parallel_world_size()==1:
18 + return input_
19 +
20 + # All-reduce.
21 + torch.distributed.all_reduce(input_, group=get_tensor_model_parallel_group())
22 +
23 + return input_
24 +
25 +
26 +def _split_along_last_dim(input_):
27 + """Split the tensor along its last dimension and keep the
28 + corresponding slice."""
29 +
30 + world_size = get_tensor_model_parallel_world_size()
31 + # Bypass the function if we are using only 1 GPU.
32 + if world_size == 1:
33 + return input_
34 +
35 + # Split along last dimension.
36 + input_list = split_tensor_along_last_dim(input_, world_size)
37 +
38 + # Note: torch.split does not create contiguous tensors by default.
39 + rank = get_tensor_model_parallel_rank()
40 + output = input_list[rank].contiguous()
41 +
42 + return output
43 +
44 +
45 +def _split_along_first_dim(input_):
46 + """Split the tensor along its first dimension and keep the
47 + corresponding slice."""
48 +
49 + world_size = get_tensor_model_parallel_world_size()
50 + # Bypass the function if we are using only 1 GPU.
51 + if world_size == 1:
52 + return input_
53 +
54 + # Split along first dimension.
55 + dim_size = input_.size()[0]
56 + assert dim_size % world_size == 0, \
57 + "First dimension of the tensor should be divisible by tensor parallel size"
58 + local_dim_size = dim_size // world_size
59 + rank = get_tensor_model_parallel_rank()
60 + dim_offset = rank * local_dim_size
61 +
62 + output = input_[dim_offset:dim_offset+local_dim_size].contiguous()
63 +
64 + return output
65 +
66 +
67 +def _gather_along_last_dim(input_):
68 + """Gather tensors and concatinate along the last dimension."""
69 +
70 + world_size = get_tensor_model_parallel_world_size()
71 + # Bypass the function if we are using only 1 GPU.
72 + if world_size == 1:
73 + return input_
74 +
75 + # Size and dimension.
76 + last_dim = input_.dim() - 1
77 + rank = get_tensor_model_parallel_rank()
78 +
79 + tensor_list = [torch.empty_like(input_) for _ in range(world_size)]
80 + tensor_list[rank] = input_
81 + torch.distributed.all_gather(tensor_list, input_, group=get_tensor_model_parallel_group())
82 +
83 + # Note: torch.cat already creates a contiguous tensor.
84 + output = torch.cat(tensor_list, dim=last_dim).contiguous()
85 +
86 + return output
87 +
88 +
89 +def _gather_along_first_dim(input_):
90 + """Gather tensors and concatinate along the first dimension."""
91 +
92 + world_size = get_tensor_model_parallel_world_size()
93 + # Bypass the function if we are using only 1 GPU.
94 + if world_size == 1:
95 + return input_
96 +
97 + dim_size = list(input_.size())
98 + dim_size[0] = dim_size[0] * world_size
99 +
100 + output = torch.empty(dim_size, dtype=input_.dtype,
101 + device=torch.cuda.current_device())
102 + torch.distributed._all_gather_base(output, input_.contiguous(),
103 + group=get_tensor_model_parallel_group())
104 +
105 + return output
106 +
107 +def _reduce_scatter_along_first_dim(input_):
108 + """Reduce-scatter the input tensor across model parallel group."""
109 + world_size = get_tensor_model_parallel_world_size()
110 + # Bypass the function if we are using only 1 GPU.
111 + if world_size == 1:
112 + return input_
113 +
114 + dim_size = list(input_.size())
115 + assert dim_size[0] % world_size == 0, \
116 + "First dimension of the tensor should be divisible by tensor parallel size"
117 +
118 + dim_size[0] = dim_size[0] // world_size
119 +
120 + output = torch.empty(dim_size, dtype=input_.dtype,
121 + device=torch.cuda.current_device())
122 + torch.distributed._reduce_scatter_base(output, input_.contiguous(),
123 + group=get_tensor_model_parallel_group())
124 + return output
125 +
126 +
127 +class _CopyToModelParallelRegion(torch.autograd.Function):
128 + """Pass the input to the model parallel region."""
129 +
130 + @staticmethod
131 + def symbolic(graph, input_):
132 + return input_
133 +
134 + @staticmethod
135 + def forward(ctx, input_):
136 + return input_
137 +
138 + @staticmethod
139 + def backward(ctx, grad_output):
140 + return _reduce(grad_output)
141 +
142 +
143 +class _ReduceFromModelParallelRegion(torch.autograd.Function):
144 + """All-reduce the input from the model parallel region."""
145 +
146 + @staticmethod
147 + def symbolic(graph, input_):
148 + return _reduce(input_)
149 +
150 + @staticmethod
151 + def forward(ctx, input_):
152 + return _reduce(input_)
153 +
154 + @staticmethod
155 + def backward(ctx, grad_output):
156 + return grad_output
157 +
158 +
159 +class _ScatterToModelParallelRegion(torch.autograd.Function):
160 + """Split the input and keep only the corresponding chuck to the rank."""
161 +
162 + @staticmethod
163 + def symbolic(graph, input_):
164 + return _split_along_last_dim(input_)
165 +
166 + @staticmethod
167 + def forward(ctx, input_):
168 + return _split_along_last_dim(input_)
169 +
170 + @staticmethod
171 + def backward(ctx, grad_output):
172 + return _gather_along_last_dim(grad_output)
173 +
174 +
175 +class _GatherFromModelParallelRegion(torch.autograd.Function):
176 + """Gather the input from model parallel region and concatinate."""
177 +
178 + @staticmethod
179 + def symbolic(graph, input_):
180 + return _gather_along_last_dim(input_)
181 +
182 + @staticmethod
183 + def forward(ctx, input_):
184 + return _gather_along_last_dim(input_)
185 +
186 + @staticmethod
187 + def backward(ctx, grad_output):
188 + return _split_along_last_dim(grad_output)
189 +
190 +
191 +class _ScatterToSequenceParallelRegion(torch.autograd.Function):
192 + """Split the input and keep only the corresponding chuck to the rank."""
193 +
194 + @staticmethod
195 + def symbolic(graph, input_):
196 + return _split_along_first_dim(input_)
197 +
198 + @staticmethod
199 + def forward(ctx, input_):
200 + return _split_along_first_dim(input_)
201 +
202 + @staticmethod
203 + def backward(ctx, grad_output):
204 + return _gather_along_first_dim(grad_output)
205 +
206 +
207 +class _GatherFromSequenceParallelRegion(torch.autograd.Function):
208 + """Gather the input from sequence parallel region and concatinate."""
209 +
210 + @staticmethod
211 + def symbolic(graph, input_, tensor_parallel_output_grad=True):
212 + return _gather_along_first_dim(input_)
213 +
214 + @staticmethod
215 + def forward(ctx, input_, tensor_parallel_output_grad=True):
216 + ctx.tensor_parallel_output_grad = tensor_parallel_output_grad
217 + return _gather_along_first_dim(input_)
218 +
219 + @staticmethod
220 + def backward(ctx, grad_output):
221 + tensor_parallel_output_grad = ctx.tensor_parallel_output_grad
222 +
223 + # If the computation graph after the gather operation is
224 + # in the tensor parallel mode, output gradients need to reduce
225 + # scattered and whereas if the computation is duplicated,
226 + # output gradients need to be scattered.
227 + if tensor_parallel_output_grad:
228 + return _reduce_scatter_along_first_dim(grad_output), None
229 + else:
230 + return _split_along_first_dim(grad_output), None
231 +
232 +
233 +class _ReduceScatterToSequenceParallelRegion(torch.autograd.Function):
234 + """Reduce scatter the input from the model parallel region."""
235 +
236 + @staticmethod
237 + def symbolic(graph, input_):
238 + return _reduce_scatter_along_first_dim(input_)
239 +
240 + @staticmethod
241 + def forward(ctx, input_):
242 + return _reduce_scatter_along_first_dim(input_)
243 +
244 + @staticmethod
245 + def backward(ctx, grad_output):
246 + return _gather_along_first_dim(grad_output)
247 +
248 +
249 +# -----------------
250 +# Helper functions.
251 +# -----------------
252 +
253 +def copy_to_tensor_model_parallel_region(input_):
254 + return _CopyToModelParallelRegion.apply(input_)
255 +
256 +
257 +def reduce_from_tensor_model_parallel_region(input_):
258 + return _ReduceFromModelParallelRegion.apply(input_)
259 +
260 +
261 +def scatter_to_tensor_model_parallel_region(input_):
262 + return _ScatterToModelParallelRegion.apply(input_)
263 +
264 +
265 +def gather_from_tensor_model_parallel_region(input_):
266 + return _GatherFromModelParallelRegion.apply(input_)
267 +
268 +
269 +def scatter_to_sequence_parallel_region(input_):
270 + return _ScatterToSequenceParallelRegion.apply(input_)
271 +
272 +
273 +def gather_from_sequence_parallel_region(input_, tensor_parallel_output_grad=True):
274 + return _GatherFromSequenceParallelRegion.apply(input_, tensor_parallel_output_grad)
275 +
276 +
277 +def reduce_scatter_to_sequence_parallel_region(input_):
278 + return _ReduceScatterToSequenceParallelRegion.apply(input_)
279 +
cacheflow/parallel_utils/tensor_parallel/random.py new
+253
@@ -0,0 +1,253 @@
1 +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2 +
3 +# Parts of the code here are adapted from PyTorch
4 +# repo: https://github.com/pytorch/pytorch
5 +
6 +import contextlib
7 +
8 +import torch
9 +from torch import _C
10 +from torch.cuda import _lazy_call, device as device_ctx_manager
11 +from torch.utils.checkpoint import detach_variable
12 +
13 +from cacheflow.parallel_utils.parallel_state import (
14 + get_data_parallel_rank,
15 + get_tensor_model_parallel_group,
16 + get_tensor_model_parallel_rank,
17 + get_tensor_model_parallel_world_size,
18 +)
19 +
20 +from .utils import (
21 + split_tensor_into_1d_equal_chunks,
22 + gather_split_1d_tensor,
23 +)
24 +
25 +from cacheflow.parallel_utils.utils import safely_set_viewless_tensor_data
26 +
27 +# Default name for the model parallel rng tracker.
28 +_MODEL_PARALLEL_RNG_TRACKER_NAME = 'model-parallel-rng'
29 +
30 +
31 +def _set_cuda_rng_state(new_state, device=-1):
32 + """Sets the random number generator state of the current GPU.
33 +
34 + Argumentss:
35 + new_state (torch.ByteTensor): The desired state
36 + This function is adapted from PyTorch repo (torch.cuda.set_rng_state)
37 + with a single change: the input state is not cloned. Cloning caused
38 + major performance issues for +4 GPU cases.
39 + """
40 + if hasattr(_C, '_cuda_setRNGState') and callable(_C._cuda_setRNGState):
41 + # older PyTorch
42 + def cb():
43 + with device_ctx_manager(device):
44 + _C._cuda_setRNGState(new_state)
45 + else:
46 + # newer PyTorch
47 + if device == -1:
48 + device = torch.device('cuda')
49 + elif isinstance(device, str):
50 + device = torch.device(device)
51 + elif isinstance(device, int):
52 + device = torch.device('cuda', device)
53 +
54 + def cb():
55 + idx = device.index
56 + if idx is None:
57 + idx = torch.cuda.current_device()
58 + default_generator = torch.cuda.default_generators[idx]
59 + default_generator.set_state(new_state)
60 +
61 + _lazy_call(cb)
62 +
63 +
64 +
65 +class CudaRNGStatesTracker:
66 + """Tracker for the cuda RNG states.
67 +
68 + Using the `add` method, a cuda rng state is initialized based on
69 + the input `seed` and is assigned to `name`. Later, by forking the
70 + rng state, we can perform operations and return to our starting
71 + cuda state.
72 + """
73 +
74 + def __init__(self):
75 + # Map from a string name to the cuda rng state.
76 + self.states_ = {}
77 + # Seeds are just for book keeping and ensure no seed is set twice.
78 + self.seeds_ = set()
79 +
80 + def reset(self):
81 + """Set to the initial state (no tracker)."""
82 + self.states_ = {}
83 + self.seeds_ = set()
84 +
85 + def get_states(self):
86 + """Get rng states. Copy the dictionary so we have direct
87 + pointers to the states, not just a pointer to the dictionary."""
88 + states = {}
89 + for name in self.states_:
90 + states[name] = self.states_[name]
91 + return states
92 +
93 + def set_states(self, states):
94 + """Set the rng states. For efficiency purposes, we do not check
95 + the size of seed for compatibility."""
96 + self.states_ = states
97 +
98 + def add(self, name, seed):
99 + """Track the rng state."""
100 + # Check seed is not already used.
101 + if seed in self.seeds_:
102 + raise Exception('seed {} already exists'.format(seed))
103 + self.seeds_.add(seed)
104 + # Check that state is not already defined.
105 + if name in self.states_:
106 + raise Exception('cuda rng state {} already exists'.format(name))
107 + # Get the current rng state.
108 + orig_rng_state = torch.cuda.get_rng_state()
109 + # Set the new state and store it.
110 + torch.cuda.manual_seed(seed)
111 + self.states_[name] = torch.cuda.get_rng_state()
112 + # Reset rng state to what it was.
113 + _set_cuda_rng_state(orig_rng_state)
114 +
115 + @contextlib.contextmanager
116 + def fork(self, name=_MODEL_PARALLEL_RNG_TRACKER_NAME):
117 + """Fork the cuda rng state, perform operations, and exit with
118 + the original state."""
119 + # Check if we have added the state
120 + if name not in self.states_:
121 + raise Exception('cuda rng state {} is not added'.format(name))
122 + # Store current rng state.
123 + orig_cuda_rng_state = torch.cuda.get_rng_state()
124 + # Set rng state to the desired one
125 + _set_cuda_rng_state(self.states_[name])
126 + # Do the stuff we wanted to do.
127 + try:
128 + yield
129 + finally:
130 + # Update the current rng state for later use.
131 + self.states_[name] = torch.cuda.get_rng_state()
132 + # And set the state to the original state we started with.
133 + _set_cuda_rng_state(orig_cuda_rng_state)
134 +
135 +
136 +# RNG tracker object.
137 +_CUDA_RNG_STATE_TRACKER = CudaRNGStatesTracker()
138 +
139 +
140 +def get_cuda_rng_tracker():
141 + """Get cuda rng tracker."""
142 + return _CUDA_RNG_STATE_TRACKER
143 +
144 +
145 +def model_parallel_cuda_manual_seed(seed):
146 + """Initialize model parallel cuda seed.
147 +
148 + This function should be called after the model parallel is
149 + initialized. Also, no torch.cuda.manual_seed should be called
150 + after this function. Basically, this is replacement for that
151 + function.
152 + Two set of RNG states are tracked:
153 + default state: This is for data parallelism and is the same among a
154 + set of model parallel GPUs but different across
155 + different model paralle groups. This is used for
156 + example for dropout in the non-tensor-model-parallel regions.
157 + tensor-model-parallel state: This state is different among a set of model
158 + parallel GPUs, but the same across data parallel
159 + groups. This is used for example for dropout in
160 + model parallel regions.
161 + """
162 + # 2718 is just for fun and any POSITIVE value will work.
163 + offset = seed + 2718
164 + tensor_model_parallel_seed = offset + get_tensor_model_parallel_rank()
165 + # Data parallel gets the original seed.
166 + data_parallel_seed = seed
167 +
168 + _CUDA_RNG_STATE_TRACKER.reset()
169 + # Set the default state.
170 + torch.cuda.manual_seed(data_parallel_seed)
171 + # and model parallel state.
172 + _CUDA_RNG_STATE_TRACKER.add(_MODEL_PARALLEL_RNG_TRACKER_NAME,
173 + tensor_model_parallel_seed)
174 +
175 +
176 +class CheckpointFunction(torch.autograd.Function):
177 + """This function is adapted from torch.utils.checkpoint with
178 + two main changes:
179 + 1) torch.cuda.set_rng_state is replaced with `_set_cuda_rng_state`
180 + 2) the states in the model parallel tracker are also properly
181 + tracked/set/reset.
182 + """
183 + @staticmethod
184 + def forward(ctx, run_function, distribute_saved_activations, *args):
185 + ctx.run_function = run_function
186 + ctx.distribute_saved_activations \
187 + = distribute_saved_activations
188 +
189 + # Copy the rng states.
190 + ctx.fwd_cpu_rng_state = torch.get_rng_state()
191 + ctx.fwd_cuda_rng_state = torch.cuda.get_rng_state()
192 + ctx.fwd_cuda_rng_state_tracker = get_cuda_rng_tracker().get_states()
193 +
194 + with torch.no_grad():
195 + outputs = run_function(*args)
196 +
197 + # Divide hidden states across model parallel group and only keep
198 + # the chunk corresponding to the current rank.
199 + if distribute_saved_activations:
200 + ctx.input_0_shape = args[0].data.shape
201 + safely_set_viewless_tensor_data(
202 + args[0],
203 + split_tensor_into_1d_equal_chunks(args[0].data, new_buffer=True))
204 +
205 + # Store everything.
206 + ctx.save_for_backward(*args)
207 +
208 + return outputs
209 +
210 + @staticmethod
211 + def backward(ctx, *args):
212 + if not torch.autograd._is_checkpoint_valid():
213 + raise RuntimeError("Checkpointing is not compatible with .grad(), "
214 + "please use .backward() if possible")
215 + inputs = ctx.saved_tensors
216 + if ctx.distribute_saved_activations:
217 + safely_set_viewless_tensor_data(
218 + inputs[0],
219 + gather_split_1d_tensor(inputs[0].data).view(ctx.input_0_shape))
220 +
221 + # Store the current states.
222 + bwd_cpu_rng_state = torch.get_rng_state()
223 + bwd_cuda_rng_state = torch.cuda.get_rng_state()
224 + bwd_cuda_rng_state_tracker = get_cuda_rng_tracker().get_states()
225 +
226 + # Set the states to what it used to be before the forward pass.
227 + torch.set_rng_state(ctx.fwd_cpu_rng_state)
228 + _set_cuda_rng_state(ctx.fwd_cuda_rng_state)
229 + get_cuda_rng_tracker().set_states(ctx.fwd_cuda_rng_state_tracker)
230 +
231 + # Compute the forward pass.
232 + detached_inputs = detach_variable(inputs)
233 + with torch.enable_grad():
234 + outputs = ctx.run_function(*detached_inputs)
235 +
236 + # Set the states back to what it was at the start of this function.
237 + torch.set_rng_state(bwd_cpu_rng_state)
238 + _set_cuda_rng_state(bwd_cuda_rng_state)
239 + get_cuda_rng_tracker().set_states(bwd_cuda_rng_state_tracker)
240 +
241 + if isinstance(outputs, torch.Tensor):
242 + outputs = (outputs,)
243 + torch.autograd.backward(outputs, args)
244 + grads = tuple(inp.grad if isinstance(inp, torch.Tensor) else inp
245 + for inp in detached_inputs)
246 + return (None, None) + grads
247 +
248 +
249 +def checkpoint(function, distribute_saved_activations, *args):
250 + """Checkpoint a model or part of the model.
251 + This has been directly copied from torch.utils.checkpoint."""
252 + return CheckpointFunction.apply(function,
253 + distribute_saved_activations, *args)
cacheflow/parallel_utils/tensor_parallel/utils.py new
+108
@@ -0,0 +1,108 @@
1 +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
2 +
3 +import torch
4 +from typing import List, Sequence
5 +
6 +from cacheflow.parallel_utils.utils import divide
7 +from cacheflow.parallel_utils import parallel_state
8 +
9 +def split_tensor_along_last_dim(
10 + tensor: torch.Tensor,
11 + num_partitions: int,
12 + contiguous_split_chunks: bool = False,
13 +) -> List[torch.Tensor]:
14 + """ Split a tensor along its last dimension.
15 +
16 + Arguments:
17 + tensor: input tensor.
18 + num_partitions: number of partitions to split the tensor
19 + contiguous_split_chunks: If True, make each chunk contiguous
20 + in memory.
21 +
22 + Returns:
23 + A list of Tensors
24 + """
25 + # Get the size and dimension.
26 + last_dim = tensor.dim() - 1
27 + last_dim_size = divide(tensor.size()[last_dim], num_partitions)
28 + # Split.
29 + tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
30 + # Note: torch.split does not create contiguous tensors by default.
31 + if contiguous_split_chunks:
32 + return tuple(chunk.contiguous() for chunk in tensor_list)
33 +
34 + return tensor_list
35 +
36 +def split_tensor_into_1d_equal_chunks(tensor, new_buffer=False):
37 + """ Break a tensor into equal 1D chunks across tensor parallel ranks.
38 +
39 + Returns a Tensor or View with this rank's portion of the data.
40 +
41 + Arguments:
42 + tensor: The tensor to split
43 +
44 + Keyword Arguments:
45 + new_buffer (bool): If True, returns a new Tensor.
46 + If False, returns a view into the existing Tensor.
47 + Default is False
48 +
49 + """
50 + partition_size = torch.numel(tensor) // \
51 + parallel_state.get_tensor_model_parallel_world_size()
52 + start_index = partition_size * parallel_state.get_tensor_model_parallel_rank()
53 + end_index = start_index + partition_size
54 + if new_buffer:
55 + data = torch.empty(partition_size, dtype=tensor.dtype,
56 + device=torch.cuda.current_device(),
57 + requires_grad=False)
58 + data.copy_(tensor.view(-1)[start_index:end_index])
59 + else:
60 + data = tensor.view(-1)[start_index:end_index]
61 + return data
62 +
63 +
64 +def gather_split_1d_tensor(tensor):
65 + """ Opposite of split_tensor_into_1d_equal_chunks. Gather values from tensor
66 + model parallel ranks.
67 +
68 + Returns a new Tensor with the gathered data.
69 +
70 + Arguments:
71 + tensor: A Tensor or view of this rank's portion of the data.
72 + """
73 + numel_gathered = torch.numel(tensor) * \
74 + parallel_state.get_tensor_model_parallel_world_size()
75 + gathered = torch.empty(numel_gathered, dtype=tensor.dtype,
76 + device=torch.cuda.current_device(),
77 + requires_grad=False)
78 + # TODO: This API is experimental in pytorch (as of Feb 2022) and
79 + # this might break in future pytorch releases. We chose this API
80 + # as opposed to torch.distributed.all_gather for efficiency reasons.
81 + # This API calls directly NCCL all-gather versus the former does
82 + # internal copies and can potentially cause slow down.
83 + torch.distributed._all_gather_base(gathered, tensor,
84 + group=parallel_state.get_tensor_model_parallel_group())
85 + return gathered
86 +
87 +
88 +class VocabUtility:
89 + """ Split the vocabulary into `world_size` chunks and return the first
90 + and last index of the vocabulary belonging to the `rank`
91 + partition: Note that indices in [fist, last)
92 +
93 + """
94 +
95 + @staticmethod
96 + def vocab_range_from_per_partition_vocab_size(
97 + per_partition_vocab_size: int, rank, world_size: int
98 + ) -> Sequence[int]:
99 + index_f = rank * per_partition_vocab_size
100 + index_l = index_f + per_partition_vocab_size
101 + return index_f, index_l
102 +
103 + @staticmethod
104 + def vocab_range_from_global_vocab_size(global_vocab_size: int, rank: int, world_size: int) -> Sequence[int]:
105 + per_partition_vocab_size = divide(global_vocab_size, world_size)
106 + return VocabUtility.vocab_range_from_per_partition_vocab_size(
107 + per_partition_vocab_size, rank, world_size
108 + )
cacheflow/parallel_utils/utils.py new
+120
@@ -0,0 +1,120 @@
1 +"""Utility functions used throughout Megatron core"""
2 +from functools import reduce
3 +import operator
4 +
5 +import torch
6 +
7 +from cacheflow.parallel_utils import parallel_state
8 +
9 +
10 +def ensure_divisibility(numerator, denominator):
11 + """Ensure that numerator is divisible by the denominator."""
12 + assert numerator % denominator == 0, "{} is not divisible by {}".format(
13 + numerator, denominator
14 + )
15 +
16 +
17 +def divide(numerator, denominator):
18 + """Ensure that numerator is divisible by the denominator and return
19 + the division value."""
20 + ensure_divisibility(numerator, denominator)
21 + return numerator // denominator
22 +
23 +
24 +class GlobalMemoryBuffer:
25 + """Global buffer to avoid dynamic memory allocations.
26 + Caller should ensure that buffers of the same name
27 + are not used concurrently."""
28 +
29 + def __init__(self):
30 + self.buffer = {}
31 +
32 + def get_tensor(self, tensor_shape, dtype, name):
33 + required_len = reduce(operator.mul, tensor_shape, 1)
34 + if self.buffer.get((name, dtype), None) is None or \
35 + self.buffer[(name, dtype)].numel() < required_len:
36 + self.buffer[(name, dtype)] = \
37 + torch.empty(required_len,
38 + dtype=dtype,
39 + device=torch.cuda.current_device(),
40 + requires_grad=False)
41 +
42 + return self.buffer[(name, dtype)][0:required_len].view(*tensor_shape)
43 +
44 +def _kernel_make_viewless_tensor(inp, requires_grad):
45 + '''Make a viewless tensor.
46 +
47 + View tensors have the undesirable side-affect of retaining a reference
48 + to the originally-viewed tensor, even after manually setting the '.data'
49 + field. This method creates a new tensor that links to the old tensor's
50 + data, without linking the viewed tensor, referenced via the '._base'
51 + field.
52 + '''
53 + out = torch.empty(
54 + (1,),
55 + dtype = inp.dtype,
56 + device = inp.device,
57 + requires_grad = requires_grad,
58 + )
59 + out.data = inp.data
60 + return out
61 +
62 +class MakeViewlessTensor(torch.autograd.Function):
63 + '''
64 + Autograd function to make a viewless tensor.
65 +
66 + This function should be used in cases where the computation graph needs
67 + to be propagated, but we only want a viewless tensor (e.g.,
68 + ParallelTransformer's hidden_states). Call this function by passing
69 + 'keep_graph = True' to 'make_viewless_tensor()'.
70 + '''
71 + @staticmethod
72 + def forward(ctx, inp, requires_grad):
73 + return _kernel_make_viewless_tensor(inp, requires_grad)
74 + @staticmethod
75 + def backward(ctx, grad_output):
76 + return grad_output, None
77 +
78 +def make_viewless_tensor(inp, requires_grad, keep_graph):
79 + '''
80 + Entry-point for creating viewless tensors.
81 +
82 + This method should be used, rather than calling 'MakeViewlessTensor'
83 + or '_kernel_make_viewless_tensor' directly. This method acts as a
84 + switch for determining if an autograd function or a regular method
85 + should be used to create the tensor.
86 + '''
87 +
88 + # return tensor as-is, if not a 'view'
89 + if inp._base is None:
90 + return inp
91 +
92 + # create viewless tensor
93 + if keep_graph:
94 + return MakeViewlessTensor.apply(inp, requires_grad)
95 + else:
96 + return _kernel_make_viewless_tensor(inp, requires_grad)
97 +
98 +def assert_viewless_tensor(tensor, extra_msg = None):
99 + '''Assert that a tensor is not a view (i.e., its '._base' field is
100 + not set).'''
101 + if isinstance(tensor, list):
102 + [ assert_viewless_tensor(t) for t in tensor ]
103 + return tensor
104 + if not isinstance(tensor, torch.Tensor):
105 + return tensor
106 + assert tensor._base is None, (
107 + "Ensure tensor._base is None before setting tensor.data or storing "
108 + "tensor to memory buffer. Otherwise, a memory leak will occur (and "
109 + "likely accumulate over iterations). %s"
110 + ) % extra_msg
111 + return tensor
112 +
113 +def safely_set_viewless_tensor_data(tensor, new_data_tensor):
114 + '''Safely set tensor's '.data' field.
115 +
116 + Check first that the tensor is viewless (i.e., '._base' not set). If not,
117 + raise an exception.
118 + '''
119 + assert_viewless_tensor(tensor, extra_msg = "FYI, tensor._base has shape %s, and new_data_tensor has shape %s." % ("--" if tensor._base is None else tensor._base.shape, new_data_tensor.shape))
120 + tensor.data = new_data_tensor
cacheflow/sequence.py
+6
@@ -158,3 +158,9 @@ class SequenceOutputs:
158 f'parent_seq_id={self.parent_seq_id}, '
159 f'output_token={self.output_token}), '
160 f'logprobs={self.logprobs}')
161 +
162 + def __eq__(self, other: 'SequenceOutputs') -> bool:
163 + return (self.seq_id == other.seq_id and
164 + self.parent_seq_id == other.parent_seq_id and
165 + self.output_token == other.output_token and
166 + self.logprobs == other.logprobs)
cacheflow/utils.py
+17
@@ -1,4 +1,11 @@
1 import enum
2 +import random
3 +
4 +import numpy as np
5 +import torch
6 +
7 +from cacheflow.parallel_utils.parallel_state import model_parallel_is_initialized
8 +from cacheflow.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
9
10
11 class Device(enum.Enum):
@@ -18,3 +25,13 @@ class Counter:
25
26 def reset(self) -> None:
27 self.counter = 0
28 +
29 +def set_random_seed(seed: int):
30 + random.seed(seed)
31 + np.random.seed(seed)
32 + torch.manual_seed(seed)
33 + if torch.cuda.is_available():
34 + torch.cuda.manual_seed_all(seed)
35 +
36 + if model_parallel_is_initialized():
37 + model_parallel_cuda_manual_seed(seed)
cacheflow/worker/cache_engine.py
+4 -6
@@ -11,7 +11,6 @@ class CacheEngine:
11 def __init__(
12 self,
13 worker_id: int,
14 - gpu_id: int,
14 num_layers: int,
15 num_heads: int,
16 head_size: int,
@@ -25,7 +24,6 @@ class CacheEngine:
24 f'head_size ({head_size}) must be a multiple of 16.')
25
26 self.worker_id = worker_id
28 - self.gpu_id = gpu_id
27 self.num_layers = num_layers
28 self.num_heads = num_heads
29 self.head_size = head_size
@@ -39,8 +37,8 @@ class CacheEngine:
37 self.cpu_cache = self.allocate_cpu_cache()
38
39 # Initialize the stream for caching operations.
42 - self.cache_stream = torch.cuda.Stream(device=gpu_id)
43 - assert self.cache_stream != torch.cuda.current_stream(device=gpu_id)
40 + self.cache_stream = torch.cuda.Stream()
41 + assert self.cache_stream != torch.cuda.current_stream()
42 # Initialize the events for stream synchronization.
43 self.events = [torch.cuda.Event() for _ in range(num_layers)]
44
@@ -69,12 +67,12 @@ class CacheEngine:
67 key_blocks = torch.empty(
68 size=(self.num_gpu_blocks, *key_block_shape),
69 dtype=self.dtype,
72 - device=self.gpu_id,
70 + device="cuda",
71 )
72 value_blocks = torch.empty(
73 size=(self.num_gpu_blocks, *value_block_shape),
74 dtype=self.dtype,
77 - device=self.gpu_id,
75 + device="cuda",
76 )
77 gpu_cache.append((key_blocks, value_blocks))
78 return gpu_cache
cacheflow/worker/controller.py
+42 -19
@@ -1,45 +1,62 @@
1 -from typing import Dict, List, Union
1 +from typing import Dict, List, Union, Tuple
2 +
3 +import ray
4
5 from cacheflow.master.scheduler import Scheduler
6 from cacheflow.sequence import SequenceGroupInputs
7 from cacheflow.worker.worker import Worker
8
9
10 +DeviceID = Tuple[int, str, int] # rank, node resource (node IP), device id
11 +
12 +
13 class Controller:
14
15 def __init__(
16 self,
12 - node_id: int,
13 - num_workers: int,
17 + stage_id: int,
18 + stage_devices: List[DeviceID],
19 + world_size: int,
20 + tensor_parallel_size: int,
21 + pipeline_parallel_size: int,
22 + distributed_init_method: str,
23 model_name: str,
24 block_size: int,
25 num_gpu_blocks: int,
26 num_cpu_blocks: int,
27 dtype: str,
28 seed: int,
29 + model_path: str,
30 ) -> None:
21 - self.node_id = node_id
22 - self.num_workers = num_workers
31 + self.stage_id = stage_id
32 + self.stage_devices = stage_devices
33 self.model_name = model_name
34 self.block_size = block_size
35 self.num_gpu_blocks = num_gpu_blocks
36 self.num_cpu_blocks = num_cpu_blocks
37
38 # Which pipeline stage is this node assigned to?
29 - self.is_first_stage = node_id == 0
39 + self.is_first_stage = stage_id == 0
40 self.is_last_stage = False
41
42 self.workers: List[Worker] = []
33 - for i in range(num_workers):
34 - worker = Worker(
35 - worker_id=node_id + i,
36 - gpu_id=i,
43 + for rank, node_resource, device_id in stage_devices:
44 + worker_cls = ray.remote(num_cpus=0,
45 + num_gpus=1,
46 + resources={node_resource: 1e-5})(Worker)
47 + worker = worker_cls.remote(
48 model_name=model_name,
49 block_size=block_size,
50 num_gpu_blocks=num_gpu_blocks,
51 num_cpu_blocks=num_cpu_blocks,
52 dtype=dtype,
53 seed=seed,
54 + distributed_init_method=distributed_init_method,
55 + rank=rank,
56 + world_size=world_size,
57 + tensor_parallel_size=tensor_parallel_size,
58 + pipeline_parallel_size=pipeline_parallel_size,
59 + model_path=model_path,
60 )
61 self.workers.append(worker)
62
@@ -57,15 +74,21 @@ class Controller:
74 blocks_to_swap_out: Dict[int, int],
75 blocks_to_copy: Dict[int, List[int]],
76 ) -> None:
60 - # FIXME: Support tensor parallelism.
61 - assert len(self.workers) == 1
62 - worker = self.workers[0]
63 - output = worker.execute_stage(
64 - input_seq_groups,
65 - blocks_to_swap_in,
66 - blocks_to_swap_out,
67 - blocks_to_copy,
68 - )
77 + futures = []
78 + for worker in self.workers:
79 + future = worker.execute_stage.remote(
80 + input_seq_groups,
81 + blocks_to_swap_in,
82 + blocks_to_swap_out,
83 + blocks_to_copy,
84 + )
85 + futures.append(future)
86 +
87 + all_outputs = ray.get(futures)
88 + # Make sure all workers have the same results.
89 + output = all_outputs[0]
90 + for other_output in all_outputs[1:]:
91 + assert output == other_output
92
93 if self.is_last_stage:
94 self.next_node.post_step(output)
cacheflow/worker/worker.py
+51 -22
@@ -3,49 +3,58 @@ from typing import Dict, List, Tuple
3 import torch
4
5 from cacheflow.models import get_model
6 -from cacheflow.models import set_seed
6 from cacheflow.models import InputMetadata
7 from cacheflow.sampling_params import SamplingParams
8 from cacheflow.sequence import SequenceGroupInputs
9 from cacheflow.sequence import SequenceOutputs
10 from cacheflow.worker.cache_engine import CacheEngine
11 +from cacheflow.parallel_utils.parallel_state import (
12 + initialize_model_parallel, get_tensor_model_parallel_world_size)
13 +from cacheflow.utils import set_random_seed
14
15
16 class Worker:
17
18 def __init__(
19 self,
18 - worker_id: int,
19 - gpu_id: int,
20 model_name: str,
21 block_size: int,
22 num_gpu_blocks: int,
23 num_cpu_blocks: int,
24 dtype: str,
25 seed: int,
26 + distributed_init_method: str,
27 + rank: int,
28 + world_size: int,
29 + model_path: str,
30 + tensor_parallel_size: int = 1,
31 + pipeline_parallel_size: int = 1,
32 ) -> None:
27 - self.worker_id = worker_id
28 - self.gpu_id = gpu_id
33 + self.init_distributed_environment(distributed_init_method,
34 + rank,
35 + world_size,
36 + tensor_parallel_size,
37 + pipeline_parallel_size)
38 + self.worker_id = rank
39 self.block_size = block_size
30 -
31 - self.device = torch.device('cuda', index=gpu_id)
40 + set_random_seed(seed)
41
42 # Initialize the model.
34 - # FIXME(woosuk): This is a hack.
35 - self.model = get_model(model_name, dtype=dtype).to(device=self.device)
43 + self.model, self.dtype = get_model(model_name, dtype=dtype, path=model_path)
44 + self.model = self.model.cuda()
45 + tensor_model_parallel_world_size = (
46 + get_tensor_model_parallel_world_size())
47 self.num_layers = self.model.config.num_hidden_layers
37 - self.num_heads = self.model.config.num_attention_heads
38 - self.head_size = self.model.config.hidden_size // self.num_heads
39 - self.dtype = self.model.dtype
48 + assert self.model.config.num_attention_heads % tensor_model_parallel_world_size == 0
49 + self.num_heads = self.model.config.num_attention_heads // tensor_model_parallel_world_size
50 + self.head_size = self.model.config.hidden_size // (self.num_heads * tensor_model_parallel_world_size)
51
41 - # Set the seed.
42 - # We set the seed after initializing the model to ensure that
52 + # We reset the seed after initializing the model to ensure that
53 # the random state is not affected by the model initialization.
44 - set_seed(seed)
54 + set_random_seed(seed)
55
56 self.cache_engine = CacheEngine(
47 - worker_id=worker_id,
48 - gpu_id=gpu_id,
57 + worker_id=self.worker_id,
58 num_layers=self.num_layers,
59 num_heads=self.num_heads,
60 head_size=self.head_size,
@@ -57,6 +66,26 @@ class Worker:
66 self.cache_events = self.cache_engine.events
67 self.gpu_cache = self.cache_engine.gpu_cache
68
69 +
70 + def init_distributed_environment(self,
71 + distributed_init_method: str,
72 + rank: int,
73 + world_size: int,
74 + tensor_parallel_size: int = 1,
75 + pipeline_parallel_size: int = 1) -> None:
76 + """Initialize the distributed environment."""
77 + torch.distributed.init_process_group(
78 + backend='nccl',
79 + init_method=distributed_init_method,
80 + world_size=world_size,
81 + rank=rank,
82 + )
83 + # A small all_reduce for warmup.
84 + torch.distributed.all_reduce(torch.zeros(1).cuda())
85 + initialize_model_parallel(tensor_parallel_size,
86 + pipeline_parallel_size)
87 +
88 +
89 def prepare_inputs(
90 self,
91 input_seq_groups: List[SequenceGroupInputs],
@@ -142,18 +171,18 @@ class Worker:
171
172 # Convert to tensors.
173 tokens_tensor = torch.tensor(
145 - input_tokens, dtype=torch.long, device=self.device)
174 + input_tokens, dtype=torch.long, device='cuda')
175 positions_tensor = torch.tensor(
147 - input_positions, dtype=torch.long, device=self.device)
176 + input_positions, dtype=torch.long, device='cuda')
177 slot_mapping_tensor = torch.tensor(
149 - slot_mapping, dtype=torch.int, device=self.device)
178 + slot_mapping, dtype=torch.int, device='cuda')
179 context_lens_tensor = torch.tensor(
151 - context_lens, dtype=torch.int, device=self.device)
180 + context_lens, dtype=torch.int, device='cuda')
181 padded_block_tables = [
182 _pad_to_max(block_table, max_num_blocks_per_seq)
183 for block_table in generation_block_tables]
184 block_tables_tensor = torch.tensor(
156 - padded_block_tables, dtype=torch.int, device=self.device)
185 + padded_block_tables, dtype=torch.int, device='cuda')
186
187 input_metadata = InputMetadata(
188 seq_groups=seq_groups,
server.py
+115 -23
@@ -1,30 +1,99 @@
1 import argparse
2 -from typing import List
2 +import random
3 +from typing import List, Tuple, Dict
4 +
5 +import ray
6
7 from cacheflow.master.frontend import Frontend
8 from cacheflow.master.scheduler import Scheduler
9 from cacheflow.models import get_memory_analyzer
7 -from cacheflow.worker.controller import Controller
8 -
9 -parser = argparse.ArgumentParser(description='CacheFlow server')
10 -parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
11 -parser.add_argument('--num-nodes', type=int, default=1, help='number of nodes')
12 -parser.add_argument('--num-workers', type=int, default=1, help='number of workers per node')
13 -parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='token block size')
14 -# NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
15 -parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
16 -# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
17 -parser.add_argument('--seed', type=int, default=0, help='random seed')
18 -parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
19 -parser.add_argument('--max-batch-size', type=int, default=2560, help='maximum number of batched tokens')
20 -args = parser.parse_args()
21 -
22 -
23 -def main():
10 +from cacheflow.worker.controller import Controller, DeviceID
11 +
12 +
13 +def initialize_ray_cluster(
14 + address: str = 'auto',
15 + pipeline_parallel_size: int = 1,
16 + tensor_parallel_size: int = 1,
17 +) -> Tuple[int, int, str, List[List[DeviceID]]]:
18 + # Connect to a ray cluster.
19 + ray.init(address=address)
20 +
21 + # Assume we have a uniform cluster that each node has the same number of
22 + # GPUs for now.
23 + valid_node_resources = []
24 + num_devices_per_node = None
25 + for node in ray.nodes():
26 + if (not node['Alive']) or node['Resources']['GPU'] <= 0:
27 + continue
28 + if num_devices_per_node is None:
29 + num_devices_per_node = node['Resources']['GPU']
30 + else:
31 + assert num_devices_per_node == node['Resources']['GPU'], (
32 + "The number of GPUs per node is not uniform.")
33 + for key in node['Resources']:
34 + if key.startswith('node:'):
35 + valid_node_resources.append(key)
36 +
37 + num_nodes = len(valid_node_resources)
38 +
39 + assert (pipeline_parallel_size * tensor_parallel_size
40 + <= num_nodes * num_devices_per_node), (
41 + "The number of required GPUs exceeds the total number of "
42 + "available GPUs.")
43 + if tensor_parallel_size >= num_devices_per_node:
44 + assert tensor_parallel_size % num_devices_per_node == 0, (
45 + "The number of tensor parallelism is not divisible by the "
46 + "number of GPUs per node.")
47 + else:
48 + assert num_devices_per_node % tensor_parallel_size == 0, (
49 + "The number of GPUs per node is not divisible by the number "
50 + "of tensor parallelism.")
51 +
52 + # Assign GPUs to pipeline stages.
53 + rank = 0
54 + current_node_id = 0
55 + current_device_id = 0
56 + distributed_init_method = None
57 + all_stage_devices = []
58 +
59 + for i in range(pipeline_parallel_size):
60 + stage_devices = []
61 + for j in range(tensor_parallel_size):
62 + node_resource = valid_node_resources[current_node_id]
63 + stage_devices.append((rank, node_resource, current_device_id))
64 + if distributed_init_method is None:
65 + ip = node_resource.split("node:")[-1]
66 + port = random.randint(10000, 20000)
67 + distributed_init_method = f"tcp://{ip}:{port}"
68 + rank += 1
69 + current_device_id += 1
70 + if current_device_id >= num_devices_per_node:
71 + current_node_id += 1
72 + current_device_id = 0
73 + all_stage_devices.append(stage_devices)
74 +
75 + return (num_nodes, num_devices_per_node, distributed_init_method,
76 + all_stage_devices)
77 +
78 +
79 +def main(args: argparse.Namespace):
80 + # TODO(zhuohan): Support pipeline parallelism.
81 + assert args.pipeline_parallel_size == 1, (
82 + 'Pipeline parallelism is not supported yet.')
83 +
84 + (num_nodes, num_devices_per_node, distributed_init_method,
85 + all_stage_devices) = (
86 + initialize_ray_cluster(
87 + pipeline_parallel_size=args.pipeline_parallel_size,
88 + tensor_parallel_size=args.tensor_parallel_size))
89 +
90 + world_size = args.pipeline_parallel_size * args.tensor_parallel_size
91 +
92 memory_analyzer = get_memory_analyzer(
93 model_name=args.model,
94 block_size=args.block_size,
95 dtype=args.dtype,
96 + tensor_parallel_size=args.tensor_parallel_size,
97 )
98 num_gpu_blocks = memory_analyzer.get_max_num_gpu_blocks(
99 max_num_batched_tokens=args.max_batch_size)
@@ -32,18 +101,23 @@ def main():
101 swap_space=args.swap_space)
102 print(f'# GPU blocks: {num_gpu_blocks}, # CPU blocks: {num_cpu_blocks}')
103
35 - # Create a controller for each node.
104 + # Create a controller for each pipeline stage.
105 controllers: List[Controller] = []
37 - for i in range(args.num_nodes):
106 + for i in range(args.pipeline_parallel_size):
107 controller = Controller(
39 - node_id=i,
40 - num_workers=args.num_workers,
108 + stage_id=i,
109 + stage_devices=all_stage_devices[i],
110 + world_size=world_size,
111 + pipeline_parallel_size=args.pipeline_parallel_size,
112 + tensor_parallel_size=args.tensor_parallel_size,
113 + distributed_init_method=distributed_init_method,
114 model_name=args.model,
115 block_size=args.block_size,
116 num_gpu_blocks=num_gpu_blocks,
117 num_cpu_blocks=num_cpu_blocks,
118 dtype=args.dtype,
119 seed=args.seed,
120 + model_path=args.model_path,
121 )
122 controllers.append(controller)
123
@@ -83,4 +157,22 @@ def main():
157
158
159 if __name__ == '__main__':
86 - main()
160 + parser = argparse.ArgumentParser(description='CacheFlow server')
161 + # Model arguments
162 + parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
163 + parser.add_argument('--model-path', type=str, default='~/.cacheflow/model_weights',
164 + help='model path to download and load the weights')
165 + # Parallel arguments
166 + parser.add_argument('--pipeline-parallel-size', type=int, default=1, help='number of pipeline stages')
167 + parser.add_argument('--tensor-parallel-size', type=int, default=1, help='number of tensor parallel replicas')
168 + # KV cache arguments
169 + parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='token block size')
170 + # NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
171 + parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
172 + # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
173 + parser.add_argument('--seed', type=int, default=0, help='random seed')
174 + parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
175 + parser.add_argument('--max-batch-size', type=int, default=2560, help='maximum number of batched tokens')
176 + args = parser.parse_args()
177 +
178 + main(args)