@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0d93f15694
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Accelerate LLaMA model loading (#234)

JFDuan committed Aug 30, 2023 at 16:00 UTC 0d93f15694273417372561e9e5c191ed9ce4d77a
8 files changed +191 -113
vllm/model_executor/models/aquila.py
+9 -14
@@ -34,8 +34,9 @@ from vllm.model_executor.input_metadata import InputMetadata
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
36 from vllm.model_executor.layers.sampler import Sampler
37 -from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
38 - load_tensor_parallel_weights)
37 +from vllm.model_executor.weight_utils import (
38 + hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
39 + load_tensor_parallel_weights)
40 from vllm.model_executor.parallel_utils.parallel_state import (
41 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -280,8 +281,7 @@ class AquilaForCausalLM(nn.Module):
281 return next_tokens
282
283 _column_parallel_weights = [
283 - "embed_tokens.weight", "lm_head.weight", "qkv_proj.weight",
284 - "gate_proj.weight", "up_proj.weight"
284 + "qkv_proj.weight", "gate_proj.weight", "up_proj.weight"
285 ]
286 _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
287
@@ -309,16 +309,6 @@ class AquilaForCausalLM(nn.Module):
309 if "rotary_emb.inv_freq" in name:
310 continue
311
312 - if "embed_tokens" in name or "lm_head" in name:
313 - param = state_dict[name]
314 - # Consider padding in the vocab size.
315 - padded_vocab_size = (param.shape[0] * tp_size)
316 - num_extra_rows = padded_vocab_size - self.config.vocab_size
317 - extra_rows = torch.empty(num_extra_rows,
318 - loaded_weight.shape[1])
319 - extra_rows = extra_rows.to(loaded_weight)
320 - loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
321 -
312 is_attention_weight = False
313 for weight_name, shard_size, offset in attention_weight_specs:
314 if weight_name not in name:
@@ -356,6 +346,11 @@ class AquilaForCausalLM(nn.Module):
346 continue
347
348 param = state_dict[name]
349 + if "embed_tokens" in name or "lm_head" in name:
350 + load_padded_tensor_parallel_vocab(param, loaded_weight,
351 + tensor_model_parallel_rank)
352 + continue
353 +
354 load_tensor_parallel_weights(param, loaded_weight, name,
355 self._column_parallel_weights,
356 self._row_parallel_weights,
vllm/model_executor/models/baichuan.py
+12 -17
@@ -32,10 +32,12 @@ from vllm.sequence import SequenceOutputs
32 from vllm.model_executor.input_metadata import InputMetadata
33 from vllm.model_executor.layers.activation import SiluAndMul
34 from vllm.model_executor.layers.layernorm import RMSNorm
35 -from vllm.model_executor.layers.attention import PagedAttentionWithRoPE, PagedAttentionWithALiBi
35 +from vllm.model_executor.layers.attention import (PagedAttentionWithRoPE,
36 + PagedAttentionWithALiBi)
37 from vllm.model_executor.layers.sampler import Sampler
37 -from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
38 - load_tensor_parallel_weights)
38 +from vllm.model_executor.weight_utils import (
39 + hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
40 + load_tensor_parallel_weights)
41 from vllm.model_executor.parallel_utils.parallel_state import (
42 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
43 from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -295,10 +297,7 @@ class BaiChuanBaseForCausalLM(nn.Module):
297 input_metadata)
298 return next_tokens
299
298 - _column_parallel_weights = [
299 - "embed_tokens.weight",
300 - "lm_head.weight",
301 - ]
300 + _column_parallel_weights = []
301 _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
302
303 def load_weights(self,
@@ -314,16 +313,6 @@ class BaiChuanBaseForCausalLM(nn.Module):
313 if "rotary_emb.inv_freq" in name:
314 continue
315
317 - if "embed_tokens" in name or "lm_head" in name:
318 - # Consider padding in the vocab size.
319 - param = state_dict[name]
320 - padded_vocab_size = param.shape[0] * tp_world_size
321 - num_extra_rows = padded_vocab_size - self.config.vocab_size
322 - extra_rows = torch.empty(num_extra_rows,
323 - loaded_weight.shape[1])
324 - extra_rows = extra_rows.to(loaded_weight)
325 - loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
326 -
316 if "W_pack" in name:
317 total_num_heads = self.config.num_attention_heads
318 hidden_size = self.config.hidden_size
@@ -355,6 +344,12 @@ class BaiChuanBaseForCausalLM(nn.Module):
344 continue
345
346 param = state_dict[name]
347 +
348 + if "embed_tokens" in name or "lm_head" in name:
349 + load_padded_tensor_parallel_vocab(param, loaded_weight,
350 + tp_rank)
351 + continue
352 +
353 load_tensor_parallel_weights(
354 param,
355 loaded_weight,
vllm/model_executor/models/gpt2.py
+7 -11
@@ -31,8 +31,9 @@ from vllm.model_executor.input_metadata import InputMetadata
31 from vllm.model_executor.layers.activation import get_act_fn
32 from vllm.model_executor.layers.attention import PagedAttention
33 from vllm.model_executor.layers.sampler import Sampler
34 -from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
35 - load_tensor_parallel_weights)
34 +from vllm.model_executor.weight_utils import (
35 + hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
36 + load_tensor_parallel_weights)
37 from vllm.model_executor.parallel_utils.parallel_state import (
38 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39 from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -224,7 +225,7 @@ class GPT2LMHeadModel(nn.Module):
225 input_metadata)
226 return next_tokens
227
227 - _column_parallel_weights = ["wte.weight", "c_fc.weight", "c_fc.bias"]
228 + _column_parallel_weights = ["c_fc.weight", "c_fc.bias"]
229 _row_parallel_weights = ["c_proj.weight"]
230
231 def load_weights(self,
@@ -261,14 +262,9 @@ class GPT2LMHeadModel(nn.Module):
262 param = state_dict[name]
263
264 if name == "transformer.wte.weight":
264 - # Consider padding in the vocab size.
265 - padded_vocab_size = (param.shape[0] *
266 - tensor_model_parallel_world_size)
267 - num_extra_rows = padded_vocab_size - self.config.vocab_size
268 - extra_rows = torch.empty(num_extra_rows,
269 - loaded_weight.shape[1])
270 - extra_rows = extra_rows.to(loaded_weight)
271 - loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
265 + load_padded_tensor_parallel_vocab(param, loaded_weight,
266 + tensor_model_parallel_rank)
267 + continue
268
269 # For the fused QKV linear layer, manually shard the weights.
270 if "c_attn" in name:
vllm/model_executor/models/gpt_bigcode.py
+7 -11
@@ -32,8 +32,9 @@ from vllm.model_executor.input_metadata import InputMetadata
32 from vllm.model_executor.layers.activation import get_act_fn
33 from vllm.model_executor.layers.attention import PagedAttention
34 from vllm.model_executor.layers.sampler import Sampler
35 -from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
36 - load_tensor_parallel_weights)
35 +from vllm.model_executor.weight_utils import (
36 + hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
37 + load_tensor_parallel_weights)
38 from vllm.model_executor.parallel_utils.parallel_state import (
39 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
40 from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -252,7 +253,7 @@ class GPTBigCodeForCausalLM(nn.Module):
253 input_metadata)
254 return next_tokens
255
255 - _column_parallel_weights = ["wte.weight", "c_fc.weight", "c_fc.bias"]
256 + _column_parallel_weights = ["c_fc.weight", "c_fc.bias"]
257 _row_parallel_weights = ["c_proj.weight"]
258
259 def load_weights(self,
@@ -328,14 +329,9 @@ class GPTBigCodeForCausalLM(nn.Module):
329 param = state_dict[name]
330
331 if name == "transformer.wte.weight":
331 - # Consider padding in the vocab size.
332 - padded_vocab_size = param.shape[
333 - 0] * tensor_model_parallel_world_size
334 - num_extra_rows = padded_vocab_size - self.config.vocab_size
335 - extra_rows = torch.empty(num_extra_rows,
336 - loaded_weight.shape[1])
337 - extra_rows = extra_rows.to(loaded_weight)
338 - loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
332 + load_padded_tensor_parallel_vocab(param, loaded_weight,
333 + tensor_model_parallel_rank)
334 + continue
335
336 load_tensor_parallel_weights(param, loaded_weight, name,
337 self._column_parallel_weights,
vllm/model_executor/models/internlm.py
+7 -14
@@ -14,8 +14,9 @@ from vllm.model_executor.parallel_utils.parallel_state import (
14 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
15 from vllm.model_executor.parallel_utils.tensor_parallel import (
16 ColumnParallelLinear, RowParallelLinear, VocabParallelEmbedding)
17 -from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
18 - load_tensor_parallel_weights)
17 +from vllm.model_executor.weight_utils import (
18 + hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
19 + load_tensor_parallel_weights)
20 from vllm.sequence import SequenceOutputs
21
22 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -225,8 +226,7 @@ class InternLMForCausalLM(nn.Module):
226 return next_tokens
227
228 _column_parallel_weights = [
228 - "embed_tokens.weight", "lm_head.weight", "qkv_proj.weight",
229 - "gate_proj.weight", "up_proj.weight"
229 + "qkv_proj.weight", "gate_proj.weight", "up_proj.weight"
230 ]
231 _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
232
@@ -234,8 +234,6 @@ class InternLMForCausalLM(nn.Module):
234 model_name_or_path: str,
235 cache_dir: Optional[str] = None,
236 use_np_cache: bool = False):
237 - tensor_model_parallel_world_size = (
238 - get_tensor_model_parallel_world_size())
237 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
238 state_dict = self.state_dict()
239
@@ -246,14 +244,9 @@ class InternLMForCausalLM(nn.Module):
244
245 if "embed_tokens" in name or "lm_head" in name:
246 param = state_dict[name]
249 - # Consider padding in the vocab size.
250 - padded_vocab_size = (param.shape[0] *
251 - tensor_model_parallel_world_size)
252 - num_extra_rows = padded_vocab_size - self.config.vocab_size
253 - extra_rows = torch.empty(num_extra_rows,
254 - loaded_weight.shape[1])
255 - extra_rows = extra_rows.to(loaded_weight)
256 - loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
247 + load_padded_tensor_parallel_vocab(param, loaded_weight,
248 + tensor_model_parallel_rank)
249 + continue
250
251 is_attention_weight = False
252 for stride_id, att_weight_name in enumerate(
vllm/model_executor/models/llama.py
+13 -16
@@ -36,8 +36,9 @@ from vllm.model_executor.layers.activation import SiluAndMul
36 from vllm.model_executor.layers.layernorm import RMSNorm
37 from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
38 from vllm.model_executor.layers.sampler import Sampler
39 -from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
40 - load_tensor_parallel_weights)
39 +from vllm.model_executor.weight_utils import (
40 + load_tensor_parallel_weights, load_padded_tensor_parallel_vocab,
41 + hf_model_weights_iterator)
42 from vllm.model_executor.parallel_utils.parallel_state import (
43 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
44 from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -263,15 +264,15 @@ class LlamaForCausalLM(nn.Module):
264 return next_tokens
265
266 _column_parallel_weights = [
266 - "embed_tokens.weight", "lm_head.weight", "qkv_proj.weight",
267 - "gate_proj.weight", "up_proj.weight"
267 + "qkv_proj.weight", "gate_proj.weight", "up_proj.weight"
268 ]
269 _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
270
271 def load_weights(self,
272 model_name_or_path: str,
273 cache_dir: Optional[str] = None,
274 - use_np_cache: bool = False):
274 + use_np_cache: bool = False,
275 + use_safetensor: bool = True):
276 tp_size = get_tensor_model_parallel_world_size()
277 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
278 q_proj_shard_size = (self.config.hidden_size // tp_size)
@@ -288,20 +289,10 @@ class LlamaForCausalLM(nn.Module):
289 state_dict = self.state_dict()
290
291 for name, loaded_weight in hf_model_weights_iterator(
291 - model_name_or_path, cache_dir, use_np_cache):
292 + model_name_or_path, cache_dir, use_np_cache, use_safetensor):
293 if "rotary_emb.inv_freq" in name:
294 continue
295
295 - if "embed_tokens" in name or "lm_head" in name:
296 - param = state_dict[name]
297 - # Consider padding in the vocab size.
298 - padded_vocab_size = (param.shape[0] * tp_size)
299 - num_extra_rows = padded_vocab_size - self.config.vocab_size
300 - extra_rows = torch.empty(num_extra_rows,
301 - loaded_weight.shape[1])
302 - extra_rows = extra_rows.to(loaded_weight)
303 - loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
304 -
296 is_attention_weight = False
297 for weight_name, shard_size, offset in attention_weight_specs:
298 if weight_name not in name:
@@ -339,6 +330,12 @@ class LlamaForCausalLM(nn.Module):
330 continue
331
332 param = state_dict[name]
333 +
334 + if "embed_tokens" in name or "lm_head" in name:
335 + load_padded_tensor_parallel_vocab(param, loaded_weight,
336 + tensor_model_parallel_rank)
337 + continue
338 +
339 load_tensor_parallel_weights(param, loaded_weight, name,
340 self._column_parallel_weights,
341 self._row_parallel_weights,
vllm/model_executor/models/qwen.py
+8 -11
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
20 from vllm.model_executor.layers.sampler import Sampler
21 from vllm.model_executor.weight_utils import (
22 hf_model_weights_iterator,
23 + load_padded_tensor_parallel_vocab,
24 load_tensor_parallel_weights,
25 )
26 from vllm.model_executor.parallel_utils.parallel_state import (
@@ -241,7 +242,7 @@ class QWenLMHeadModel(nn.Module):
242 input_metadata)
243 return next_tokens
244
244 - _column_parallel_weights = ["wte.weight", "lm_head.weight"]
245 + _column_parallel_weights = []
246 _row_parallel_weights = ["c_proj.weight"]
247
248 def load_weights(
@@ -259,16 +260,6 @@ class QWenLMHeadModel(nn.Module):
260 if "rotary_emb.inv_freq" in name:
261 continue
262
262 - if "wte" in name or "lm_head" in name:
263 - # Consider padding in the vocab size.
264 - param = state_dict[name]
265 - padded_vocab_size = param.shape[0] * tp_world_size
266 - num_extra_rows = padded_vocab_size - self.config.vocab_size
267 - extra_rows = torch.empty(num_extra_rows,
268 - loaded_weight.shape[1])
269 - extra_rows = extra_rows.to(loaded_weight)
270 - loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
271 -
263 if "c_attn" in name:
264 total_num_heads = self.config.num_attention_heads
265 hidden_size = self.config.hidden_size
@@ -306,6 +297,12 @@ class QWenLMHeadModel(nn.Module):
297 continue
298
299 param = state_dict[name]
300 +
301 + if "wte" in name or "lm_head" in name:
302 + load_padded_tensor_parallel_vocab(param, loaded_weight,
303 + tp_rank)
304 + continue
305 +
306 load_tensor_parallel_weights(
307 param,
308 loaded_weight,
vllm/model_executor/weight_utils.py
+128 -19
@@ -3,13 +3,19 @@ import filelock
3 import glob
4 import json
5 import os
6 -from typing import Iterator, List, Optional, Tuple
6 +from collections import defaultdict
7 +from typing import Iterator, List, Optional, Tuple, Any
8
9 from huggingface_hub import snapshot_download
10 +from safetensors.torch import load_file, save_file, safe_open
11 import numpy as np
12 import torch
13 from tqdm.auto import tqdm
14
15 +from vllm.logger import init_logger
16 +
17 +logger = init_logger(__name__)
18 +
19
20 class Disabledtqdm(tqdm):
21
@@ -17,43 +23,118 @@ class Disabledtqdm(tqdm):
23 super().__init__(*args, **kwargs, disable=True)
24
25
20 -def hf_model_weights_iterator(
21 - model_name_or_path: str,
22 - cache_dir: Optional[str] = None,
23 - use_np_cache: bool = False,
24 -) -> Iterator[Tuple[str, torch.Tensor]]:
25 - # Prepare file lock directory to prevent multiple processes from
26 - # downloading the same model weights at the same time.
26 +def get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):
27 lock_dir = cache_dir if cache_dir is not None else "/tmp"
28 lock_file_name = model_name_or_path.replace("/", "-") + ".lock"
29 lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name))
30 -
30 + return lock
31 +
32 +
33 +def _shared_pointers(tensors):
34 + ptrs = defaultdict(list)
35 + for k, v in tensors.items():
36 + ptrs[v.data_ptr()].append(k)
37 + failing = []
38 + for _, names in ptrs.items():
39 + if len(names) > 1:
40 + failing.append(names)
41 + return failing
42 +
43 +
44 +def convert_bin_to_safetensor_file(
45 + pt_filename: str,
46 + sf_filename: str,
47 +):
48 + loaded = torch.load(pt_filename, map_location="cpu")
49 + if "state_dict" in loaded:
50 + loaded = loaded["state_dict"]
51 + shared = _shared_pointers(loaded)
52 + for shared_weights in shared:
53 + for name in shared_weights[1:]:
54 + loaded.pop(name)
55 +
56 + # For tensors to be contiguous
57 + loaded = {k: v.contiguous() for k, v in loaded.items()}
58 +
59 + dirname = os.path.dirname(sf_filename)
60 + os.makedirs(dirname, exist_ok=True)
61 + save_file(loaded, sf_filename, metadata={"format": "pt"})
62 +
63 + # check file size
64 + sf_size = os.stat(sf_filename).st_size
65 + pt_size = os.stat(pt_filename).st_size
66 + if (sf_size - pt_size) / pt_size > 0.01:
67 + raise RuntimeError(f"""The file size different is more than 1%:
68 + - {sf_filename}: {sf_size}
69 + - {pt_filename}: {pt_size}
70 + """)
71 +
72 + # check if the tensors are the same
73 + reloaded = load_file(sf_filename)
74 + for k in loaded:
75 + pt_tensor = loaded[k]
76 + sf_tensor = reloaded[k]
77 + if not torch.equal(pt_tensor, sf_tensor):
78 + raise RuntimeError(f"The output tensors do not match for key {k}")
79 +
80 +
81 +def prepare_hf_model_weights(
82 + model_name_or_path: str,
83 + cache_dir: Optional[str] = None,
84 + use_safetensor: bool = False,
85 +):
86 # Download model weights from huggingface.
87 is_local = os.path.isdir(model_name_or_path)
88 + allow_patterns = "*.safetensors" if use_safetensor else "*.bin"
89 if not is_local:
34 - with lock:
90 + # Use file lock to prevent multiple processes from
91 + # downloading the same model weights at the same time.
92 + with get_lock(model_name_or_path, cache_dir):
93 hf_folder = snapshot_download(model_name_or_path,
36 - allow_patterns="*.bin",
94 + allow_patterns=allow_patterns,
95 cache_dir=cache_dir,
96 tqdm_class=Disabledtqdm)
97 else:
98 hf_folder = model_name_or_path
99 + hf_weights_files = glob.glob(os.path.join(hf_folder, allow_patterns))
100 + if not use_safetensor:
101 + hf_weights_files = [
102 + x for x in hf_weights_files if not x.endswith("training_args.bin")
103 + ]
104 +
105 + if len(hf_weights_files) == 0 and use_safetensor:
106 + logger.warning("No *.safetensors files found, "
107 + "fall back to *.bin files")
108 + return prepare_hf_model_weights(model_name_or_path,
109 + cache_dir=cache_dir,
110 + use_safetensor=False)
111 + return hf_folder, hf_weights_files, use_safetensor
112
42 - hf_bin_files = [
43 - x for x in glob.glob(os.path.join(hf_folder, "*.bin"))
44 - if not x.endswith("training_args.bin")
45 - ]
113 +
114 +def hf_model_weights_iterator(
115 + model_name_or_path: str,
116 + cache_dir: Optional[str] = None,
117 + use_np_cache: bool = False,
118 + use_safetensor: bool = False,
119 +) -> Iterator[Tuple[str, torch.Tensor]]:
120 + hf_folder, hf_weights_files, use_safetensor = prepare_hf_model_weights(
121 + model_name_or_path, cache_dir=cache_dir, use_safetensor=use_safetensor)
122
123 if use_np_cache:
124 + # Currently np_cache only support *.bin checkpoints
125 + assert use_safetensor is False
126 +
127 # Convert the model weights from torch tensors to numpy arrays for
128 # faster loading.
129 np_folder = os.path.join(hf_folder, "np")
130 os.makedirs(np_folder, exist_ok=True)
131 weight_names_file = os.path.join(np_folder, "weight_names.json")
53 - with lock:
132 + # Use file lock to prevent multiple processes from
133 + # dumping the same model weights to numpy at the same time.
134 + with get_lock(model_name_or_path, cache_dir):
135 if not os.path.exists(weight_names_file):
136 weight_names = []
56 - for bin_file in hf_bin_files:
137 + for bin_file in hf_weights_files:
138 state = torch.load(bin_file, map_location="cpu")
139 for name, param in state.items():
140 param_path = os.path.join(np_folder, name)
@@ -71,8 +152,14 @@ def hf_model_weights_iterator(
152 with open(param_path, "rb") as f:
153 param = np.load(f)
154 yield name, torch.from_numpy(param)
155 + elif use_safetensor:
156 + for st_file in hf_weights_files:
157 + with safe_open(st_file, framework="pt") as f:
158 + for name in f.keys():
159 + param = f.get_slice(name)
160 + yield name, param
161 else:
75 - for bin_file in hf_bin_files:
162 + for bin_file in hf_weights_files:
163 state = torch.load(bin_file, map_location="cpu")
164 for name, param in state.items():
165 yield name, param
@@ -80,9 +167,26 @@ def hf_model_weights_iterator(
167 torch.cuda.empty_cache()
168
169
170 +def load_padded_tensor_parallel_vocab(
171 + param: torch.Tensor,
172 + loaded_weight: Any, # `torch.Tensor` or `PySafeSlice`
173 + tensor_model_parallel_rank: int,
174 +) -> None:
175 + shard_size = param.shape[0]
176 + start_idx = tensor_model_parallel_rank * shard_size
177 + end_idx = (tensor_model_parallel_rank + 1) * shard_size
178 + loaded_weight = loaded_weight[start_idx:end_idx]
179 +
180 + # convert PySafeSlice object to torch.Tensor
181 + if not isinstance(loaded_weight, torch.Tensor):
182 + loaded_weight = loaded_weight[:]
183 +
184 + param[:loaded_weight.shape[0]].copy_(loaded_weight)
185 +
186 +
187 def load_tensor_parallel_weights(
188 param: torch.Tensor,
85 - loaded_weight: torch.Tensor,
189 + loaded_weight: Any, # `torch.Tensor` or `PySafeSlice`
190 param_name: str,
191 column_parallel_weight_names: List[str],
192 row_parallel_weight_names: List[str],
@@ -102,6 +206,11 @@ def load_tensor_parallel_weights(
206 end_idx = (tensor_model_parallel_rank + 1) * shard_size
207 loaded_weight = loaded_weight[:, start_idx:end_idx]
208 break
209 +
210 + # convert PySafeSlice object to torch.Tensor
211 + if not isinstance(loaded_weight, torch.Tensor):
212 + loaded_weight = loaded_weight[:]
213 +
214 assert param.shape == loaded_weight.shape, (
215 f"{param_name} shape mismatch between model and checkpoint: "
216 f"{param.shape} != {loaded_weight.shape}")