A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Model] add minicpm (#3893)
ywfang committed
Apr 8, 2024 at 18:28 UTC
b4543c8f6bf67a7f1a0d6d0fd6cf5697c7eeaabb
4 files changed
+543
README.md
+1
@@ -70,6 +70,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
70
- InternLM2 (`internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc.)
71
- Jais (`core42/jais-13b`, `core42/jais-13b-chat`, `core42/jais-30b-v3`, `core42/jais-30b-chat-v3`, etc.)
72
- LLaMA & LLaMA-2 (`meta-llama/Llama-2-70b-hf`, `lmsys/vicuna-13b-v1.3`, `young-geng/koala`, `openlm-research/open_llama_13b`, etc.)
73
+- MiniCPM (`openbmb/MiniCPM-2B-sft-bf16`, `openbmb/MiniCPM-2B-dpo-bf16`, etc.)
74
- Mistral (`mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc.)
75
- Mixtral (`mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, etc.)
76
- MPT (`mosaicml/mpt-7b`, `mosaicml/mpt-30b`, etc.)
docs/source/models/supported_models.rst
+4
@@ -83,6 +83,10 @@ Alongside each architecture, we include some popular models that use it.
83
- LLaMA, LLaMA-2, Vicuna, Alpaca, Yi
84
- :code:`meta-llama/Llama-2-13b-hf`, :code:`meta-llama/Llama-2-70b-hf`, :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`01-ai/Yi-6B`, :code:`01-ai/Yi-34B`, etc.
85
- ✅︎
86
+ * - :code:`MiniCPMForCausalLM`
87
+ - MiniCPM
88
+ - :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, etc.
89
+ -
90
* - :code:`MistralForCausalLM`
91
- Mistral, Mistral-Instruct
92
- :code:`mistralai/Mistral-7B-v0.1`, :code:`mistralai/Mistral-7B-Instruct-v0.1`, etc.
vllm/model_executor/models/__init__.py
+1
@@ -41,6 +41,7 @@ _MODELS = {
41
# transformers's mpt class has lower case
42
"MptForCausalLM": ("mpt", "MPTForCausalLM"),
43
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),
44
+ "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
45
"OLMoForCausalLM": ("olmo", "OLMoForCausalLM"),
46
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
47
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
vllm/model_executor/models/minicpm.py
new
+537
@@ -0,0 +1,537 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only MiniCPM model compatible with HuggingFace weights."""
24
+import math
25
+from typing import Any, Dict, List, Optional, Tuple
26
+
27
+import torch
28
+from torch import nn
29
+
30
+from vllm.attention import Attention, AttentionMetadata
31
+from vllm.config import LoRAConfig
32
+from vllm.model_executor.layers.activation import SiluAndMul
33
+from vllm.model_executor.layers.fused_moe import fused_moe
34
+from vllm.model_executor.layers.layernorm import RMSNorm
35
+from vllm.model_executor.layers.linear import (LinearMethodBase,
36
+ MergedColumnParallelLinear,
37
+ QKVParallelLinear,
38
+ ReplicatedLinear,
39
+ RowParallelLinear)
40
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
41
+from vllm.model_executor.layers.rotary_embedding import get_rope
42
+from vllm.model_executor.layers.sampler import Sampler
43
+from vllm.model_executor.layers.vocab_parallel_embedding import (
44
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45
+from vllm.model_executor.parallel_utils.communication_op import (
46
+ tensor_model_parallel_all_reduce)
47
+from vllm.model_executor.parallel_utils.parallel_state import (
48
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
49
+from vllm.model_executor.sampling_metadata import SamplingMetadata
50
+from vllm.model_executor.utils import set_weight_attrs
51
+from vllm.model_executor.weight_utils import (default_weight_loader,
52
+ hf_model_weights_iterator)
53
+from vllm.sequence import SamplerOutput
54
+
55
+
56
+class MiniCPMMoE(nn.Module):
57
+ """A tensor-parallel MoE implementation that shards each expert
58
+ across all ranks.
59
+
60
+ Each expert's weights are sharded across all ranks and a fused MoE
61
+ kernel is used for the forward pass, and finally we reduce the outputs
62
+ across ranks.
63
+ """
64
+
65
+ def __init__(
66
+ self,
67
+ num_experts: int,
68
+ top_k: int,
69
+ hidden_size: int,
70
+ intermediate_size: int,
71
+ params_dtype: Optional[torch.dtype] = None,
72
+ tp_size: Optional[int] = None,
73
+ ):
74
+ super().__init__()
75
+ self.tp_size = tp_size or get_tensor_model_parallel_world_size()
76
+ self.num_total_experts = num_experts
77
+ self.top_k = top_k
78
+ self.hidden_size = hidden_size
79
+ self.intermediate_size = intermediate_size // self.tp_size
80
+
81
+ if params_dtype is None:
82
+ params_dtype = torch.get_default_dtype()
83
+ self.params_dtype = params_dtype
84
+
85
+ self.gate = ReplicatedLinear(self.hidden_size,
86
+ self.num_total_experts,
87
+ bias=False,
88
+ params_dtype=self.params_dtype,
89
+ linear_method=None)
90
+
91
+ self.ws = nn.Parameter(
92
+ torch.empty(self.num_total_experts,
93
+ 2 * self.intermediate_size,
94
+ self.hidden_size,
95
+ device="cuda",
96
+ dtype=self.params_dtype))
97
+ self.w2s = nn.Parameter(
98
+ torch.empty(self.num_total_experts,
99
+ self.hidden_size,
100
+ self.intermediate_size,
101
+ device="cuda",
102
+ dtype=self.params_dtype))
103
+
104
+ set_weight_attrs(self.ws, {
105
+ "weight_loader": self.weight_loader,
106
+ })
107
+ set_weight_attrs(self.w2s, {
108
+ "weight_loader": self.weight_loader,
109
+ })
110
+
111
+ def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
112
+ weight_name: str, expert_id: int):
113
+ tp_rank = get_tensor_model_parallel_rank()
114
+ param_data = param.data
115
+ shard_size = self.intermediate_size
116
+ shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
117
+ if weight_name.endswith("w1.weight"):
118
+ param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :]
119
+ if weight_name.endswith("w3.weight"):
120
+ param_data[expert_id,
121
+ shard_size:2 * shard_size, :] = loaded_weight[shard, :]
122
+ if weight_name.endswith("w2.weight"):
123
+ param_data[expert_id, :, :] = loaded_weight[:, shard]
124
+
125
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
126
+ num_tokens, hidden_size = hidden_states.shape
127
+ hidden_states = hidden_states.view(-1, self.hidden_size)
128
+ # router_logits: (num_tokens, n_experts)
129
+ router_logits, _ = self.gate(hidden_states)
130
+ final_hidden_states = fused_moe(hidden_states,
131
+ self.ws,
132
+ self.w2s,
133
+ router_logits,
134
+ self.top_k,
135
+ renormalize=True,
136
+ inplace=True)
137
+
138
+ if self.tp_size > 1:
139
+ final_hidden_states = tensor_model_parallel_all_reduce(
140
+ final_hidden_states)
141
+
142
+ return final_hidden_states.view(num_tokens, hidden_size)
143
+
144
+
145
+class MiniCPMMLP(nn.Module):
146
+
147
+ def __init__(
148
+ self,
149
+ hidden_size: int,
150
+ intermediate_size: int,
151
+ hidden_act: str,
152
+ linear_method: Optional[LinearMethodBase] = None,
153
+ ) -> None:
154
+ super().__init__()
155
+ self.gate_up_proj = MergedColumnParallelLinear(
156
+ hidden_size, [intermediate_size] * 2,
157
+ bias=False,
158
+ linear_method=linear_method)
159
+ self.down_proj = RowParallelLinear(intermediate_size,
160
+ hidden_size,
161
+ bias=False,
162
+ linear_method=linear_method)
163
+ if hidden_act != "silu":
164
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
165
+ "Only silu is supported for now.")
166
+ self.act_fn = SiluAndMul()
167
+
168
+ def forward(self, x):
169
+ gate_up, _ = self.gate_up_proj(x)
170
+ x = self.act_fn(gate_up)
171
+ x, _ = self.down_proj(x)
172
+ return x
173
+
174
+
175
+class MiniCPMAttention(nn.Module):
176
+
177
+ def __init__(
178
+ self,
179
+ hidden_size: int,
180
+ num_heads: int,
181
+ num_kv_heads: int,
182
+ rope_theta: float = 10000,
183
+ rope_scaling: Optional[Dict[str, Any]] = None,
184
+ max_position_embeddings: int = 8192,
185
+ linear_method: Optional[LinearMethodBase] = None,
186
+ ) -> None:
187
+ super().__init__()
188
+ self.hidden_size = hidden_size
189
+ tp_size = get_tensor_model_parallel_world_size()
190
+ self.total_num_heads = num_heads
191
+ assert self.total_num_heads % tp_size == 0
192
+ self.num_heads = self.total_num_heads // tp_size
193
+ self.total_num_kv_heads = num_kv_heads
194
+ if self.total_num_kv_heads >= tp_size:
195
+ # Number of KV heads is greater than TP size, so we partition
196
+ # the KV heads across multiple tensor parallel GPUs.
197
+ assert self.total_num_kv_heads % tp_size == 0
198
+ else:
199
+ # Number of KV heads is less than TP size, so we replicate
200
+ # the KV heads across multiple tensor parallel GPUs.
201
+ assert tp_size % self.total_num_kv_heads == 0
202
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
203
+ self.head_dim = hidden_size // self.total_num_heads
204
+ self.q_size = self.num_heads * self.head_dim
205
+ self.kv_size = self.num_kv_heads * self.head_dim
206
+ self.scaling = self.head_dim**-0.5
207
+ self.rope_theta = rope_theta
208
+ self.max_position_embeddings = max_position_embeddings
209
+
210
+ self.qkv_proj = QKVParallelLinear(
211
+ hidden_size,
212
+ self.head_dim,
213
+ self.total_num_heads,
214
+ self.total_num_kv_heads,
215
+ bias=False,
216
+ linear_method=linear_method,
217
+ )
218
+ self.o_proj = RowParallelLinear(
219
+ self.total_num_heads * self.head_dim,
220
+ hidden_size,
221
+ bias=False,
222
+ linear_method=linear_method,
223
+ )
224
+
225
+ self.rotary_emb = get_rope(
226
+ self.head_dim,
227
+ rotary_dim=self.head_dim,
228
+ max_position=max_position_embeddings,
229
+ base=rope_theta,
230
+ rope_scaling=rope_scaling,
231
+ )
232
+ # set rope as fp32 instead of bf16
233
+ self.rotary_emb.cos_sin_cache = self.rotary_emb._compute_cos_sin_cache(
234
+ )
235
+ self.attn = Attention(self.num_heads,
236
+ self.head_dim,
237
+ self.scaling,
238
+ num_kv_heads=self.num_kv_heads)
239
+
240
+ def forward(
241
+ self,
242
+ positions: torch.Tensor,
243
+ hidden_states: torch.Tensor,
244
+ kv_cache: torch.Tensor,
245
+ attn_metadata: AttentionMetadata,
246
+ ) -> torch.Tensor:
247
+ qkv, _ = self.qkv_proj(hidden_states)
248
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
249
+ orig_dtype = q.dtype
250
+ q, k = q.float(), k.float()
251
+ q, k = self.rotary_emb(positions, q, k)
252
+ q, k = q.to(orig_dtype), k.to(orig_dtype)
253
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
254
+ output, _ = self.o_proj(attn_output)
255
+ return output
256
+
257
+
258
+class MiniCPMDecoderLayer(nn.Module):
259
+
260
+ def __init__(
261
+ self,
262
+ config,
263
+ linear_method: Optional[LinearMethodBase] = None,
264
+ ) -> None:
265
+ super().__init__()
266
+ self.config = config
267
+ self.hidden_size = config.hidden_size
268
+ rope_theta = getattr(config, "rope_theta", 10000)
269
+ rope_scaling = getattr(config, "rope_scaling", None)
270
+ max_position_embeddings = getattr(config, "max_position_embeddings",
271
+ 8192)
272
+ self.self_attn = MiniCPMAttention(
273
+ hidden_size=self.hidden_size,
274
+ num_heads=config.num_attention_heads,
275
+ num_kv_heads=config.num_key_value_heads,
276
+ rope_theta=rope_theta,
277
+ rope_scaling=rope_scaling,
278
+ max_position_embeddings=max_position_embeddings,
279
+ linear_method=linear_method,
280
+ )
281
+ self.num_experts = getattr(self.config, "num_experts", 0)
282
+ if self.num_experts == 0:
283
+ self.mlp = MiniCPMMLP(
284
+ hidden_size=self.hidden_size,
285
+ intermediate_size=config.intermediate_size,
286
+ hidden_act=config.hidden_act,
287
+ linear_method=linear_method,
288
+ )
289
+ else:
290
+ self.mlp = MiniCPMMoE(num_experts=config.num_experts,
291
+ top_k=config.num_experts_per_tok,
292
+ hidden_size=config.hidden_size,
293
+ intermediate_size=config.intermediate_size)
294
+ self.input_layernorm = RMSNorm(config.hidden_size,
295
+ eps=config.rms_norm_eps)
296
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
297
+ eps=config.rms_norm_eps)
298
+
299
+ def forward(
300
+ self,
301
+ positions: torch.Tensor,
302
+ hidden_states: torch.Tensor,
303
+ kv_cache: torch.Tensor,
304
+ attn_metadata: AttentionMetadata,
305
+ residual: Optional[torch.Tensor],
306
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
307
+ # Self Attention
308
+ residual = hidden_states
309
+ hidden_states = self.input_layernorm(hidden_states)
310
+ hidden_states = self.self_attn(
311
+ positions=positions,
312
+ hidden_states=hidden_states,
313
+ kv_cache=kv_cache,
314
+ attn_metadata=attn_metadata,
315
+ )
316
+ hidden_states = residual + hidden_states * \
317
+ (self.config.scale_depth / math.sqrt(self.config.num_hidden_layers))
318
+
319
+ # Fully Connected
320
+ residual = hidden_states
321
+ hidden_states = self.post_attention_layernorm(hidden_states)
322
+ hidden_states = self.mlp(hidden_states)
323
+ hidden_states = residual + hidden_states * \
324
+ (self.config.scale_depth / math.sqrt(self.config.num_hidden_layers))
325
+
326
+ return hidden_states, None
327
+
328
+
329
+class MiniCPMModel(nn.Module):
330
+
331
+ def __init__(
332
+ self,
333
+ config,
334
+ linear_method: Optional[LinearMethodBase] = None,
335
+ lora_config: Optional[LoRAConfig] = None,
336
+ ) -> None:
337
+ super().__init__()
338
+ self.config = config
339
+ self.padding_idx = config.pad_token_id
340
+ lora_vocab = (lora_config.lora_extra_vocab_size *
341
+ (lora_config.max_loras or 1)) if lora_config else 0
342
+ self.vocab_size = config.vocab_size + lora_vocab
343
+ self.org_vocab_size = config.vocab_size
344
+ self.embed_tokens = VocabParallelEmbedding(
345
+ self.vocab_size,
346
+ config.hidden_size,
347
+ org_num_embeddings=config.vocab_size,
348
+ )
349
+ self.layers = nn.ModuleList([
350
+ MiniCPMDecoderLayer(config, linear_method)
351
+ for _ in range(config.num_hidden_layers)
352
+ ])
353
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
354
+
355
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
356
+ embedding = self.embed_tokens(input_ids)
357
+ return embedding * self.config.scale_emb
358
+
359
+ def forward(
360
+ self,
361
+ input_ids: torch.Tensor,
362
+ positions: torch.Tensor,
363
+ kv_caches: List[torch.Tensor],
364
+ attn_metadata: AttentionMetadata,
365
+ inputs_embeds: Optional[torch.Tensor] = None,
366
+ ) -> torch.Tensor:
367
+ if inputs_embeds is not None:
368
+ hidden_states = inputs_embeds
369
+ else:
370
+ hidden_states = self.get_input_embeddings(input_ids)
371
+ residual = None
372
+
373
+ for i in range(len(self.layers)):
374
+ layer = self.layers[i]
375
+ hidden_states, residual = layer(
376
+ positions,
377
+ hidden_states,
378
+ kv_caches[i],
379
+ attn_metadata,
380
+ residual,
381
+ )
382
+ hidden_states = self.norm(hidden_states)
383
+ return hidden_states
384
+
385
+
386
+class MiniCPMForCausalLM(nn.Module):
387
+ packed_modules_mapping = {
388
+ "qkv_proj": [
389
+ "q_proj",
390
+ "k_proj",
391
+ "v_proj",
392
+ ],
393
+ "gate_up_proj": [
394
+ "gate_proj",
395
+ "up_proj",
396
+ ],
397
+ }
398
+
399
+ # LoRA specific attributes
400
+ supported_lora_modules = [
401
+ "qkv_proj",
402
+ "o_proj",
403
+ "gate_up_proj",
404
+ "down_proj",
405
+ "embed_tokens",
406
+ "lm_head",
407
+ ]
408
+ embedding_modules = {
409
+ "embed_tokens": "input_embeddings",
410
+ "lm_head": "output_embeddings",
411
+ }
412
+ embedding_padding_modules = ["lm_head"]
413
+
414
+ def __init__(
415
+ self,
416
+ config,
417
+ linear_method: Optional[LinearMethodBase] = None,
418
+ lora_config: Optional[LoRAConfig] = None,
419
+ ) -> None:
420
+ super().__init__()
421
+ self.config = config
422
+ self.num_experts = getattr(self.config, "num_experts", 0)
423
+ self.linear_method = linear_method
424
+ self.model = MiniCPMModel(config,
425
+ linear_method,
426
+ lora_config=lora_config)
427
+ unpadded_vocab_size = config.vocab_size
428
+ if lora_config:
429
+ unpadded_vocab_size += lora_config.lora_extra_vocab_size
430
+ if not self.config.tie_word_embeddings:
431
+ self.lm_head = ParallelLMHead(
432
+ unpadded_vocab_size,
433
+ config.hidden_size,
434
+ org_num_embeddings=config.vocab_size,
435
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE
436
+ # We need bigger padding if using lora for kernel
437
+ # compatibility
438
+ if not lora_config else lora_config.lora_vocab_padding_size,
439
+ )
440
+ self.scale_width = self.config.hidden_size / self.config.dim_model_base
441
+
442
+ self.logits_processor = LogitsProcessor(unpadded_vocab_size,
443
+ config.vocab_size)
444
+ self.sampler = Sampler()
445
+
446
+ def forward(
447
+ self,
448
+ input_ids: torch.Tensor,
449
+ positions: torch.Tensor,
450
+ kv_caches: List[torch.Tensor],
451
+ attn_metadata: AttentionMetadata,
452
+ ) -> torch.Tensor:
453
+ hidden_states = self.model(input_ids, positions, kv_caches,
454
+ attn_metadata)
455
+ return hidden_states
456
+
457
+ def compute_logits(self, hidden_states: torch.Tensor,
458
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
459
+ hidden_states = hidden_states / self.scale_width
460
+ if self.config.tie_word_embeddings:
461
+ lm_head_weight = self.model.embed_tokens.weight
462
+ else:
463
+ lm_head_weight = self.lm_head.weight
464
+ logits = self.logits_processor(lm_head_weight, hidden_states,
465
+ sampling_metadata)
466
+ return logits
467
+
468
+ def sample(
469
+ self,
470
+ logits: torch.Tensor,
471
+ sampling_metadata: SamplingMetadata,
472
+ ) -> Optional[SamplerOutput]:
473
+ next_tokens = self.sampler(logits, sampling_metadata)
474
+ return next_tokens
475
+
476
+ def load_weights(self,
477
+ model_name_or_path: str,
478
+ cache_dir: Optional[str] = None,
479
+ load_format: str = "auto",
480
+ revision: Optional[str] = None):
481
+ stacked_params_mapping = [
482
+ # (param_name, shard_name, shard_id)
483
+ ("qkv_proj", "q_proj", "q"),
484
+ ("qkv_proj", "k_proj", "k"),
485
+ ("qkv_proj", "v_proj", "v"),
486
+ ("gate_up_proj", "gate_proj", 0),
487
+ ("gate_up_proj", "up_proj", 1),
488
+ ]
489
+ expert_params_mapping = [
490
+ # (param_name, weight_name, expert_id)
491
+ ("ws" if weight_name in ["w1", "w3"] else "w2s",
492
+ f"experts.{expert_id}.{weight_name}.weight", expert_id)
493
+ for expert_id in range(self.num_experts)
494
+ for weight_name in ["w1", "w2", "w3"]
495
+ ]
496
+ params_dict = dict(self.named_parameters())
497
+ for name, loaded_weight in hf_model_weights_iterator(
498
+ model_name_or_path, cache_dir, load_format, revision):
499
+ if "rotary_emb.inv_freq" in name:
500
+ continue
501
+ if ("rotary_emb.cos_cached" in name
502
+ or "rotary_emb.sin_cached" in name):
503
+ # Models trained using ColossalAI may include these tensors in
504
+ # the checkpoint. Skip them.
505
+ continue
506
+
507
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
508
+ if weight_name not in name:
509
+ continue
510
+ name = name.replace(weight_name, param_name)
511
+ # Skip loading extra bias for GPTQ models.
512
+ if name.endswith(".bias") and name not in params_dict:
513
+ continue
514
+ param = params_dict[name]
515
+ weight_loader = param.weight_loader
516
+ weight_loader(param, loaded_weight, shard_id)
517
+ break
518
+ else:
519
+ for param_name, weight_name, expert_id in expert_params_mapping:
520
+ if weight_name not in name:
521
+ continue
522
+ name = name.replace(weight_name, param_name)
523
+ param = params_dict[name]
524
+ weight_loader = param.weight_loader
525
+ weight_loader(param,
526
+ loaded_weight,
527
+ weight_name,
528
+ expert_id=expert_id)
529
+ break
530
+ else:
531
+ # Skip loading extra bias for GPTQ models.
532
+ if name.endswith(".bias") and name not in params_dict:
533
+ continue
534
+ param = params_dict[name]
535
+ weight_loader = getattr(param, "weight_loader",
536
+ default_weight_loader)
537
+ weight_loader(param, loaded_weight)