A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Accelerate LLaMA model loading (#234)
JFDuan committed
Aug 30, 2023 at 16:00 UTC
0d93f15694273417372561e9e5c191ed9ce4d77a
8 files changed
+191
-113
vllm/model_executor/models/aquila.py
+9
-14
@@ -34,8 +34,9 @@ from vllm.model_executor.input_metadata import InputMetadata
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
36
from vllm.model_executor.layers.sampler import Sampler
37
-from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
38
- load_tensor_parallel_weights)
37
+from vllm.model_executor.weight_utils import (
38
+ hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
39
+ load_tensor_parallel_weights)
40
from vllm.model_executor.parallel_utils.parallel_state import (
41
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -280,8 +281,7 @@ class AquilaForCausalLM(nn.Module):
281
return next_tokens
282
283
_column_parallel_weights = [
283
- "embed_tokens.weight", "lm_head.weight", "qkv_proj.weight",
284
- "gate_proj.weight", "up_proj.weight"
284
+ "qkv_proj.weight", "gate_proj.weight", "up_proj.weight"
285
]
286
_row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
287
@@ -309,16 +309,6 @@ class AquilaForCausalLM(nn.Module):
309
if "rotary_emb.inv_freq" in name:
310
continue
311
312
- if "embed_tokens" in name or "lm_head" in name:
313
- param = state_dict[name]
314
- # Consider padding in the vocab size.
315
- padded_vocab_size = (param.shape[0] * tp_size)
316
- num_extra_rows = padded_vocab_size - self.config.vocab_size
317
- extra_rows = torch.empty(num_extra_rows,
318
- loaded_weight.shape[1])
319
- extra_rows = extra_rows.to(loaded_weight)
320
- loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
321
-
312
is_attention_weight = False
313
for weight_name, shard_size, offset in attention_weight_specs:
314
if weight_name not in name:
@@ -356,6 +346,11 @@ class AquilaForCausalLM(nn.Module):
346
continue
347
348
param = state_dict[name]
349
+ if "embed_tokens" in name or "lm_head" in name:
350
+ load_padded_tensor_parallel_vocab(param, loaded_weight,
351
+ tensor_model_parallel_rank)
352
+ continue
353
+
354
load_tensor_parallel_weights(param, loaded_weight, name,
355
self._column_parallel_weights,
356
self._row_parallel_weights,
vllm/model_executor/models/baichuan.py
+12
-17
@@ -32,10 +32,12 @@ from vllm.sequence import SequenceOutputs
32
from vllm.model_executor.input_metadata import InputMetadata
33
from vllm.model_executor.layers.activation import SiluAndMul
34
from vllm.model_executor.layers.layernorm import RMSNorm
35
-from vllm.model_executor.layers.attention import PagedAttentionWithRoPE, PagedAttentionWithALiBi
35
+from vllm.model_executor.layers.attention import (PagedAttentionWithRoPE,
36
+ PagedAttentionWithALiBi)
37
from vllm.model_executor.layers.sampler import Sampler
37
-from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
38
- load_tensor_parallel_weights)
38
+from vllm.model_executor.weight_utils import (
39
+ hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
40
+ load_tensor_parallel_weights)
41
from vllm.model_executor.parallel_utils.parallel_state import (
42
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
43
from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -295,10 +297,7 @@ class BaiChuanBaseForCausalLM(nn.Module):
297
input_metadata)
298
return next_tokens
299
298
- _column_parallel_weights = [
299
- "embed_tokens.weight",
300
- "lm_head.weight",
301
- ]
300
+ _column_parallel_weights = []
301
_row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
302
303
def load_weights(self,
@@ -314,16 +313,6 @@ class BaiChuanBaseForCausalLM(nn.Module):
313
if "rotary_emb.inv_freq" in name:
314
continue
315
317
- if "embed_tokens" in name or "lm_head" in name:
318
- # Consider padding in the vocab size.
319
- param = state_dict[name]
320
- padded_vocab_size = param.shape[0] * tp_world_size
321
- num_extra_rows = padded_vocab_size - self.config.vocab_size
322
- extra_rows = torch.empty(num_extra_rows,
323
- loaded_weight.shape[1])
324
- extra_rows = extra_rows.to(loaded_weight)
325
- loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
326
-
316
if "W_pack" in name:
317
total_num_heads = self.config.num_attention_heads
318
hidden_size = self.config.hidden_size
@@ -355,6 +344,12 @@ class BaiChuanBaseForCausalLM(nn.Module):
344
continue
345
346
param = state_dict[name]
347
+
348
+ if "embed_tokens" in name or "lm_head" in name:
349
+ load_padded_tensor_parallel_vocab(param, loaded_weight,
350
+ tp_rank)
351
+ continue
352
+
353
load_tensor_parallel_weights(
354
param,
355
loaded_weight,
vllm/model_executor/models/gpt2.py
+7
-11
@@ -31,8 +31,9 @@ from vllm.model_executor.input_metadata import InputMetadata
31
from vllm.model_executor.layers.activation import get_act_fn
32
from vllm.model_executor.layers.attention import PagedAttention
33
from vllm.model_executor.layers.sampler import Sampler
34
-from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
35
- load_tensor_parallel_weights)
34
+from vllm.model_executor.weight_utils import (
35
+ hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
36
+ load_tensor_parallel_weights)
37
from vllm.model_executor.parallel_utils.parallel_state import (
38
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39
from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -224,7 +225,7 @@ class GPT2LMHeadModel(nn.Module):
225
input_metadata)
226
return next_tokens
227
227
- _column_parallel_weights = ["wte.weight", "c_fc.weight", "c_fc.bias"]
228
+ _column_parallel_weights = ["c_fc.weight", "c_fc.bias"]
229
_row_parallel_weights = ["c_proj.weight"]
230
231
def load_weights(self,
@@ -261,14 +262,9 @@ class GPT2LMHeadModel(nn.Module):
262
param = state_dict[name]
263
264
if name == "transformer.wte.weight":
264
- # Consider padding in the vocab size.
265
- padded_vocab_size = (param.shape[0] *
266
- tensor_model_parallel_world_size)
267
- num_extra_rows = padded_vocab_size - self.config.vocab_size
268
- extra_rows = torch.empty(num_extra_rows,
269
- loaded_weight.shape[1])
270
- extra_rows = extra_rows.to(loaded_weight)
271
- loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
265
+ load_padded_tensor_parallel_vocab(param, loaded_weight,
266
+ tensor_model_parallel_rank)
267
+ continue
268
269
# For the fused QKV linear layer, manually shard the weights.
270
if "c_attn" in name:
vllm/model_executor/models/gpt_bigcode.py
+7
-11
@@ -32,8 +32,9 @@ from vllm.model_executor.input_metadata import InputMetadata
32
from vllm.model_executor.layers.activation import get_act_fn
33
from vllm.model_executor.layers.attention import PagedAttention
34
from vllm.model_executor.layers.sampler import Sampler
35
-from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
36
- load_tensor_parallel_weights)
35
+from vllm.model_executor.weight_utils import (
36
+ hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
37
+ load_tensor_parallel_weights)
38
from vllm.model_executor.parallel_utils.parallel_state import (
39
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
40
from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -252,7 +253,7 @@ class GPTBigCodeForCausalLM(nn.Module):
253
input_metadata)
254
return next_tokens
255
255
- _column_parallel_weights = ["wte.weight", "c_fc.weight", "c_fc.bias"]
256
+ _column_parallel_weights = ["c_fc.weight", "c_fc.bias"]
257
_row_parallel_weights = ["c_proj.weight"]
258
259
def load_weights(self,
@@ -328,14 +329,9 @@ class GPTBigCodeForCausalLM(nn.Module):
329
param = state_dict[name]
330
331
if name == "transformer.wte.weight":
331
- # Consider padding in the vocab size.
332
- padded_vocab_size = param.shape[
333
- 0] * tensor_model_parallel_world_size
334
- num_extra_rows = padded_vocab_size - self.config.vocab_size
335
- extra_rows = torch.empty(num_extra_rows,
336
- loaded_weight.shape[1])
337
- extra_rows = extra_rows.to(loaded_weight)
338
- loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
332
+ load_padded_tensor_parallel_vocab(param, loaded_weight,
333
+ tensor_model_parallel_rank)
334
+ continue
335
336
load_tensor_parallel_weights(param, loaded_weight, name,
337
self._column_parallel_weights,
vllm/model_executor/models/internlm.py
+7
-14
@@ -14,8 +14,9 @@ from vllm.model_executor.parallel_utils.parallel_state import (
14
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
15
from vllm.model_executor.parallel_utils.tensor_parallel import (
16
ColumnParallelLinear, RowParallelLinear, VocabParallelEmbedding)
17
-from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
18
- load_tensor_parallel_weights)
17
+from vllm.model_executor.weight_utils import (
18
+ hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
19
+ load_tensor_parallel_weights)
20
from vllm.sequence import SequenceOutputs
21
22
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -225,8 +226,7 @@ class InternLMForCausalLM(nn.Module):
226
return next_tokens
227
228
_column_parallel_weights = [
228
- "embed_tokens.weight", "lm_head.weight", "qkv_proj.weight",
229
- "gate_proj.weight", "up_proj.weight"
229
+ "qkv_proj.weight", "gate_proj.weight", "up_proj.weight"
230
]
231
_row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
232
@@ -234,8 +234,6 @@ class InternLMForCausalLM(nn.Module):
234
model_name_or_path: str,
235
cache_dir: Optional[str] = None,
236
use_np_cache: bool = False):
237
- tensor_model_parallel_world_size = (
238
- get_tensor_model_parallel_world_size())
237
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
238
state_dict = self.state_dict()
239
@@ -246,14 +244,9 @@ class InternLMForCausalLM(nn.Module):
244
245
if "embed_tokens" in name or "lm_head" in name:
246
param = state_dict[name]
249
- # Consider padding in the vocab size.
250
- padded_vocab_size = (param.shape[0] *
251
- tensor_model_parallel_world_size)
252
- num_extra_rows = padded_vocab_size - self.config.vocab_size
253
- extra_rows = torch.empty(num_extra_rows,
254
- loaded_weight.shape[1])
255
- extra_rows = extra_rows.to(loaded_weight)
256
- loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
247
+ load_padded_tensor_parallel_vocab(param, loaded_weight,
248
+ tensor_model_parallel_rank)
249
+ continue
250
251
is_attention_weight = False
252
for stride_id, att_weight_name in enumerate(
vllm/model_executor/models/llama.py
+13
-16
@@ -36,8 +36,9 @@ from vllm.model_executor.layers.activation import SiluAndMul
36
from vllm.model_executor.layers.layernorm import RMSNorm
37
from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
38
from vllm.model_executor.layers.sampler import Sampler
39
-from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
40
- load_tensor_parallel_weights)
39
+from vllm.model_executor.weight_utils import (
40
+ load_tensor_parallel_weights, load_padded_tensor_parallel_vocab,
41
+ hf_model_weights_iterator)
42
from vllm.model_executor.parallel_utils.parallel_state import (
43
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
44
from vllm.model_executor.parallel_utils.tensor_parallel import (
@@ -263,15 +264,15 @@ class LlamaForCausalLM(nn.Module):
264
return next_tokens
265
266
_column_parallel_weights = [
266
- "embed_tokens.weight", "lm_head.weight", "qkv_proj.weight",
267
- "gate_proj.weight", "up_proj.weight"
267
+ "qkv_proj.weight", "gate_proj.weight", "up_proj.weight"
268
]
269
_row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
270
271
def load_weights(self,
272
model_name_or_path: str,
273
cache_dir: Optional[str] = None,
274
- use_np_cache: bool = False):
274
+ use_np_cache: bool = False,
275
+ use_safetensor: bool = True):
276
tp_size = get_tensor_model_parallel_world_size()
277
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
278
q_proj_shard_size = (self.config.hidden_size // tp_size)
@@ -288,20 +289,10 @@ class LlamaForCausalLM(nn.Module):
289
state_dict = self.state_dict()
290
291
for name, loaded_weight in hf_model_weights_iterator(
291
- model_name_or_path, cache_dir, use_np_cache):
292
+ model_name_or_path, cache_dir, use_np_cache, use_safetensor):
293
if "rotary_emb.inv_freq" in name:
294
continue
295
295
- if "embed_tokens" in name or "lm_head" in name:
296
- param = state_dict[name]
297
- # Consider padding in the vocab size.
298
- padded_vocab_size = (param.shape[0] * tp_size)
299
- num_extra_rows = padded_vocab_size - self.config.vocab_size
300
- extra_rows = torch.empty(num_extra_rows,
301
- loaded_weight.shape[1])
302
- extra_rows = extra_rows.to(loaded_weight)
303
- loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
304
-
296
is_attention_weight = False
297
for weight_name, shard_size, offset in attention_weight_specs:
298
if weight_name not in name:
@@ -339,6 +330,12 @@ class LlamaForCausalLM(nn.Module):
330
continue
331
332
param = state_dict[name]
333
+
334
+ if "embed_tokens" in name or "lm_head" in name:
335
+ load_padded_tensor_parallel_vocab(param, loaded_weight,
336
+ tensor_model_parallel_rank)
337
+ continue
338
+
339
load_tensor_parallel_weights(param, loaded_weight, name,
340
self._column_parallel_weights,
341
self._row_parallel_weights,
vllm/model_executor/models/qwen.py
+8
-11
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
20
from vllm.model_executor.layers.sampler import Sampler
21
from vllm.model_executor.weight_utils import (
22
hf_model_weights_iterator,
23
+ load_padded_tensor_parallel_vocab,
24
load_tensor_parallel_weights,
25
)
26
from vllm.model_executor.parallel_utils.parallel_state import (
@@ -241,7 +242,7 @@ class QWenLMHeadModel(nn.Module):
242
input_metadata)
243
return next_tokens
244
244
- _column_parallel_weights = ["wte.weight", "lm_head.weight"]
245
+ _column_parallel_weights = []
246
_row_parallel_weights = ["c_proj.weight"]
247
248
def load_weights(
@@ -259,16 +260,6 @@ class QWenLMHeadModel(nn.Module):
260
if "rotary_emb.inv_freq" in name:
261
continue
262
262
- if "wte" in name or "lm_head" in name:
263
- # Consider padding in the vocab size.
264
- param = state_dict[name]
265
- padded_vocab_size = param.shape[0] * tp_world_size
266
- num_extra_rows = padded_vocab_size - self.config.vocab_size
267
- extra_rows = torch.empty(num_extra_rows,
268
- loaded_weight.shape[1])
269
- extra_rows = extra_rows.to(loaded_weight)
270
- loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
271
-
263
if "c_attn" in name:
264
total_num_heads = self.config.num_attention_heads
265
hidden_size = self.config.hidden_size
@@ -306,6 +297,12 @@ class QWenLMHeadModel(nn.Module):
297
continue
298
299
param = state_dict[name]
300
+
301
+ if "wte" in name or "lm_head" in name:
302
+ load_padded_tensor_parallel_vocab(param, loaded_weight,
303
+ tp_rank)
304
+ continue
305
+
306
load_tensor_parallel_weights(
307
param,
308
loaded_weight,
vllm/model_executor/weight_utils.py
+128
-19
@@ -3,13 +3,19 @@ import filelock
3
import glob
4
import json
5
import os
6
-from typing import Iterator, List, Optional, Tuple
6
+from collections import defaultdict
7
+from typing import Iterator, List, Optional, Tuple, Any
8
9
from huggingface_hub import snapshot_download
10
+from safetensors.torch import load_file, save_file, safe_open
11
import numpy as np
12
import torch
13
from tqdm.auto import tqdm
14
15
+from vllm.logger import init_logger
16
+
17
+logger = init_logger(__name__)
18
+
19
20
class Disabledtqdm(tqdm):
21
@@ -17,43 +23,118 @@ class Disabledtqdm(tqdm):
23
super().__init__(*args, **kwargs, disable=True)
24
25
20
-def hf_model_weights_iterator(
21
- model_name_or_path: str,
22
- cache_dir: Optional[str] = None,
23
- use_np_cache: bool = False,
24
-) -> Iterator[Tuple[str, torch.Tensor]]:
25
- # Prepare file lock directory to prevent multiple processes from
26
- # downloading the same model weights at the same time.
26
+def get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):
27
lock_dir = cache_dir if cache_dir is not None else "/tmp"
28
lock_file_name = model_name_or_path.replace("/", "-") + ".lock"
29
lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name))
30
-
30
+ return lock
31
+
32
+
33
+def _shared_pointers(tensors):
34
+ ptrs = defaultdict(list)
35
+ for k, v in tensors.items():
36
+ ptrs[v.data_ptr()].append(k)
37
+ failing = []
38
+ for _, names in ptrs.items():
39
+ if len(names) > 1:
40
+ failing.append(names)
41
+ return failing
42
+
43
+
44
+def convert_bin_to_safetensor_file(
45
+ pt_filename: str,
46
+ sf_filename: str,
47
+):
48
+ loaded = torch.load(pt_filename, map_location="cpu")
49
+ if "state_dict" in loaded:
50
+ loaded = loaded["state_dict"]
51
+ shared = _shared_pointers(loaded)
52
+ for shared_weights in shared:
53
+ for name in shared_weights[1:]:
54
+ loaded.pop(name)
55
+
56
+ # For tensors to be contiguous
57
+ loaded = {k: v.contiguous() for k, v in loaded.items()}
58
+
59
+ dirname = os.path.dirname(sf_filename)
60
+ os.makedirs(dirname, exist_ok=True)
61
+ save_file(loaded, sf_filename, metadata={"format": "pt"})
62
+
63
+ # check file size
64
+ sf_size = os.stat(sf_filename).st_size
65
+ pt_size = os.stat(pt_filename).st_size
66
+ if (sf_size - pt_size) / pt_size > 0.01:
67
+ raise RuntimeError(f"""The file size different is more than 1%:
68
+ - {sf_filename}: {sf_size}
69
+ - {pt_filename}: {pt_size}
70
+ """)
71
+
72
+ # check if the tensors are the same
73
+ reloaded = load_file(sf_filename)
74
+ for k in loaded:
75
+ pt_tensor = loaded[k]
76
+ sf_tensor = reloaded[k]
77
+ if not torch.equal(pt_tensor, sf_tensor):
78
+ raise RuntimeError(f"The output tensors do not match for key {k}")
79
+
80
+
81
+def prepare_hf_model_weights(
82
+ model_name_or_path: str,
83
+ cache_dir: Optional[str] = None,
84
+ use_safetensor: bool = False,
85
+):
86
# Download model weights from huggingface.
87
is_local = os.path.isdir(model_name_or_path)
88
+ allow_patterns = "*.safetensors" if use_safetensor else "*.bin"
89
if not is_local:
34
- with lock:
90
+ # Use file lock to prevent multiple processes from
91
+ # downloading the same model weights at the same time.
92
+ with get_lock(model_name_or_path, cache_dir):
93
hf_folder = snapshot_download(model_name_or_path,
36
- allow_patterns="*.bin",
94
+ allow_patterns=allow_patterns,
95
cache_dir=cache_dir,
96
tqdm_class=Disabledtqdm)
97
else:
98
hf_folder = model_name_or_path
99
+ hf_weights_files = glob.glob(os.path.join(hf_folder, allow_patterns))
100
+ if not use_safetensor:
101
+ hf_weights_files = [
102
+ x for x in hf_weights_files if not x.endswith("training_args.bin")
103
+ ]
104
+
105
+ if len(hf_weights_files) == 0 and use_safetensor:
106
+ logger.warning("No *.safetensors files found, "
107
+ "fall back to *.bin files")
108
+ return prepare_hf_model_weights(model_name_or_path,
109
+ cache_dir=cache_dir,
110
+ use_safetensor=False)
111
+ return hf_folder, hf_weights_files, use_safetensor
112
42
- hf_bin_files = [
43
- x for x in glob.glob(os.path.join(hf_folder, "*.bin"))
44
- if not x.endswith("training_args.bin")
45
- ]
113
+
114
+def hf_model_weights_iterator(
115
+ model_name_or_path: str,
116
+ cache_dir: Optional[str] = None,
117
+ use_np_cache: bool = False,
118
+ use_safetensor: bool = False,
119
+) -> Iterator[Tuple[str, torch.Tensor]]:
120
+ hf_folder, hf_weights_files, use_safetensor = prepare_hf_model_weights(
121
+ model_name_or_path, cache_dir=cache_dir, use_safetensor=use_safetensor)
122
123
if use_np_cache:
124
+ # Currently np_cache only support *.bin checkpoints
125
+ assert use_safetensor is False
126
+
127
# Convert the model weights from torch tensors to numpy arrays for
128
# faster loading.
129
np_folder = os.path.join(hf_folder, "np")
130
os.makedirs(np_folder, exist_ok=True)
131
weight_names_file = os.path.join(np_folder, "weight_names.json")
53
- with lock:
132
+ # Use file lock to prevent multiple processes from
133
+ # dumping the same model weights to numpy at the same time.
134
+ with get_lock(model_name_or_path, cache_dir):
135
if not os.path.exists(weight_names_file):
136
weight_names = []
56
- for bin_file in hf_bin_files:
137
+ for bin_file in hf_weights_files:
138
state = torch.load(bin_file, map_location="cpu")
139
for name, param in state.items():
140
param_path = os.path.join(np_folder, name)
@@ -71,8 +152,14 @@ def hf_model_weights_iterator(
152
with open(param_path, "rb") as f:
153
param = np.load(f)
154
yield name, torch.from_numpy(param)
155
+ elif use_safetensor:
156
+ for st_file in hf_weights_files:
157
+ with safe_open(st_file, framework="pt") as f:
158
+ for name in f.keys():
159
+ param = f.get_slice(name)
160
+ yield name, param
161
else:
75
- for bin_file in hf_bin_files:
162
+ for bin_file in hf_weights_files:
163
state = torch.load(bin_file, map_location="cpu")
164
for name, param in state.items():
165
yield name, param
@@ -80,9 +167,26 @@ def hf_model_weights_iterator(
167
torch.cuda.empty_cache()
168
169
170
+def load_padded_tensor_parallel_vocab(
171
+ param: torch.Tensor,
172
+ loaded_weight: Any, # `torch.Tensor` or `PySafeSlice`
173
+ tensor_model_parallel_rank: int,
174
+) -> None:
175
+ shard_size = param.shape[0]
176
+ start_idx = tensor_model_parallel_rank * shard_size
177
+ end_idx = (tensor_model_parallel_rank + 1) * shard_size
178
+ loaded_weight = loaded_weight[start_idx:end_idx]
179
+
180
+ # convert PySafeSlice object to torch.Tensor
181
+ if not isinstance(loaded_weight, torch.Tensor):
182
+ loaded_weight = loaded_weight[:]
183
+
184
+ param[:loaded_weight.shape[0]].copy_(loaded_weight)
185
+
186
+
187
def load_tensor_parallel_weights(
188
param: torch.Tensor,
85
- loaded_weight: torch.Tensor,
189
+ loaded_weight: Any, # `torch.Tensor` or `PySafeSlice`
190
param_name: str,
191
column_parallel_weight_names: List[str],
192
row_parallel_weight_names: List[str],
@@ -102,6 +206,11 @@ def load_tensor_parallel_weights(
206
end_idx = (tensor_model_parallel_rank + 1) * shard_size
207
loaded_weight = loaded_weight[:, start_idx:end_idx]
208
break
209
+
210
+ # convert PySafeSlice object to torch.Tensor
211
+ if not isinstance(loaded_weight, torch.Tensor):
212
+ loaded_weight = loaded_weight[:]
213
+
214
assert param.shape == loaded_weight.shape, (
215
f"{param_name} shape mismatch between model and checkpoint: "
216
f"{param.shape} != {loaded_weight.shape}")