@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b5f882cc98
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Mixtral 8x7B support (#2011)

Co-authored-by: Pierre Stock <p@mistral.ai> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

Pierre Stock committed Dec 11, 2023 at 10:09 UTC b5f882cc98e2c9c6dde7357dbac2ec0c2c57d8cd
4 files changed +538
README.md
+1
@@ -60,6 +60,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
60 - InternLM (`internlm/internlm-7b`, `internlm/internlm-chat-7b`, etc.)
61 - LLaMA & LLaMA-2 (`meta-llama/Llama-2-70b-hf`, `lmsys/vicuna-13b-v1.3`, `young-geng/koala`, `openlm-research/open_llama_13b`, etc.)
62 - Mistral (`mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc.)
63 +- Mixtral (`mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, etc.)
64 - MPT (`mosaicml/mpt-7b`, `mosaicml/mpt-30b`, etc.)
65 - OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
66 - Phi-1.5 (`microsoft/phi-1_5`, etc.)
vllm/model_executor/model_loader.py
+1
@@ -33,6 +33,7 @@ _MODEL_REGISTRY = {
33 "LlamaForCausalLM": LlamaForCausalLM,
34 "LLaMAForCausalLM": LlamaForCausalLM, # For decapoda-research/llama-*
35 "MistralForCausalLM": MistralForCausalLM,
36 + "MixtralForCausalLM": MixtralForCausalLM,
37 # transformers's mpt class has lower case
38 "MptForCausalLM": MPTForCausalLM,
39 "MPTForCausalLM": MPTForCausalLM,
vllm/model_executor/models/__init__.py
+2
@@ -10,6 +10,7 @@ from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
10 from vllm.model_executor.models.internlm import InternLMForCausalLM
11 from vllm.model_executor.models.llama import LlamaForCausalLM
12 from vllm.model_executor.models.mistral import MistralForCausalLM
13 +from vllm.model_executor.models.mixtral import MixtralForCausalLM
14 from vllm.model_executor.models.mpt import MPTForCausalLM
15 from vllm.model_executor.models.opt import OPTForCausalLM
16 from vllm.model_executor.models.phi_1_5 import PhiForCausalLM
@@ -35,5 +36,6 @@ __all__ = [
36 "PhiForCausalLM",
37 "QWenLMHeadModel",
38 "MistralForCausalLM",
39 + "MixtralForCausalLM",
40 "YiForCausalLM",
41 ]
vllm/model_executor/models/mixtral.py new
+534
@@ -0,0 +1,534 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only Mixtral model."""
24 +from typing import List, Optional, Tuple, Union
25 +
26 +import numpy as np
27 +
28 +import torch
29 +import torch.nn.functional as F
30 +
31 +from torch import nn
32 +from transformers import MistralConfig
33 +
34 +try:
35 + import megablocks.ops as ops
36 +except ImportError:
37 + print("MegaBlocks not found, please see "
38 + "https://github.com/stanford-futuredata/megablocks/. "
39 + "Note that MegaBlocks depends on mosaicml-turbo, which only "
40 + "supports python 3.10.")
41 +try:
42 + import stk
43 +except ImportError:
44 + print(
45 + "STK not found: please see https://github.com/stanford-futuredata/stk")
46 +
47 +from vllm.model_executor.input_metadata import InputMetadata
48 +from vllm.model_executor.layers.attention import PagedAttention
49 +from vllm.model_executor.layers.layernorm import RMSNorm
50 +from vllm.model_executor.layers.linear import (LinearMethodBase,
51 + QKVParallelLinear,
52 + RowParallelLinear)
53 +from vllm.model_executor.layers.rotary_embedding import get_rope
54 +from vllm.model_executor.layers.sampler import Sampler
55 +from vllm.model_executor.layers.vocab_parallel_embedding import (
56 + VocabParallelEmbedding, ParallelLMHead)
57 +from vllm.model_executor.parallel_utils.communication_op import (
58 + tensor_model_parallel_all_reduce)
59 +from vllm.model_executor.parallel_utils.parallel_state import (
60 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
61 +from vllm.model_executor.sampling_metadata import SamplingMetadata
62 +from vllm.model_executor.weight_utils import (default_weight_loader,
63 + hf_model_weights_iterator)
64 +from vllm.model_executor.utils import set_weight_attrs
65 +from vllm.sequence import SamplerOutput
66 +
67 +KVCache = Tuple[torch.Tensor, torch.Tensor]
68 +
69 +
70 +def promote_scalar(x: torch.Tensor) -> torch.Tensor:
71 + return x.view(1) if len(x.size()) == 0 else x
72 +
73 +
74 +class MixtralAttention(nn.Module):
75 +
76 + def __init__(self,
77 + hidden_size: int,
78 + num_heads: int,
79 + num_kv_heads: int,
80 + max_position: int = 4096 * 32,
81 + rope_theta: float = 10000,
82 + sliding_window: Optional[int] = None) -> None:
83 + super().__init__()
84 + self.hidden_size = hidden_size
85 + tp_size = get_tensor_model_parallel_world_size()
86 + self.total_num_heads = num_heads
87 + assert self.total_num_heads % tp_size == 0
88 + self.num_heads = self.total_num_heads // tp_size
89 + self.total_num_kv_heads = num_kv_heads
90 + if self.total_num_kv_heads >= tp_size:
91 + # Number of KV heads is greater than TP size, so we partition
92 + # the KV heads across multiple tensor parallel GPUs.
93 + assert self.total_num_kv_heads % tp_size == 0
94 + else:
95 + # Number of KV heads is less than TP size, so we replicate
96 + # the KV heads across multiple tensor parallel GPUs.
97 + assert tp_size % self.total_num_kv_heads == 0
98 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
99 + self.head_dim = hidden_size // self.total_num_heads
100 + self.q_size = self.num_heads * self.head_dim
101 + self.kv_size = self.num_kv_heads * self.head_dim
102 + self.scaling = self.head_dim**-0.5
103 + self.rope_theta = rope_theta
104 + self.sliding_window = sliding_window
105 +
106 + self.wqkv = QKVParallelLinear(
107 + hidden_size,
108 + self.head_dim,
109 + self.total_num_heads,
110 + self.total_num_kv_heads,
111 + bias=False,
112 + )
113 + self.wo = RowParallelLinear(
114 + self.total_num_heads * self.head_dim,
115 + hidden_size,
116 + bias=False,
117 + )
118 + self.rotary_emb = get_rope(
119 + self.head_dim,
120 + rotary_dim=self.head_dim,
121 + max_position=max_position,
122 + base=int(self.rope_theta),
123 + is_neox_style=False, # weights not in HF format
124 + )
125 + self.attn = PagedAttention(
126 + self.num_heads,
127 + self.head_dim,
128 + self.scaling,
129 + num_kv_heads=self.num_kv_heads,
130 + sliding_window=self.sliding_window,
131 + )
132 +
133 + def forward(
134 + self,
135 + positions: torch.Tensor,
136 + hidden_states: torch.Tensor,
137 + kv_cache: KVCache,
138 + input_metadata: InputMetadata,
139 + cache_event: Optional[torch.cuda.Event],
140 + ) -> torch.Tensor:
141 + qkv, _ = self.wqkv(hidden_states)
142 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
143 + q, k = self.rotary_emb(positions, q, k)
144 + k_cache, v_cache = kv_cache
145 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
146 + cache_event)
147 + output, _ = self.wo(attn_output)
148 + return output
149 +
150 +
151 +class BlockSparseMoE(nn.Module):
152 + """
153 + Built on the paper and library Megablocks as described in
154 + https://arxiv.org/abs/2211.15841. This implementation is
155 + strictly equivalent to standard MoE with full capacity (no
156 + dropped tokens). It's faster since it formulates MoE operations
157 + in terms of block-sparse operations to accomodate imbalanced
158 + assignments of tokens to experts, whereas standard MoE either
159 + (1) drop tokens at the cost of reduced performance or (2) set
160 + capacity factor to number of experts and thus waste computation
161 + and memory on padding.
162 + """
163 +
164 + def __init__(self, hidden_dim: int, ffn_dim: int, num_experts: int,
165 + top_k: int):
166 + super().__init__()
167 + self.hidden_dim = hidden_dim
168 + self.ffn_dim = ffn_dim
169 + self.num_experts = num_experts
170 + self.top_k = top_k
171 +
172 + # gating
173 + self.gate = nn.Linear(self.hidden_dim,
174 + self.num_experts,
175 + bias=False,
176 + device=torch.cuda.current_device())
177 +
178 + tp_size = get_tensor_model_parallel_world_size()
179 + assert self.ffn_dim % tp_size == 0
180 + self.ffn_dim_per_partition = self.ffn_dim // tp_size
181 + # merged expert weights, all of size (ffn_dim * n_experts, model_dim)
182 + self.w1 = nn.Parameter(
183 + torch.empty(self.ffn_dim_per_partition * self.num_experts,
184 + self.hidden_dim,
185 + device=torch.cuda.current_device()))
186 + set_weight_attrs(self.w1, {"weight_loader": self.moe_weight_loader})
187 + self.w2 = nn.Parameter(
188 + torch.empty(self.ffn_dim_per_partition * self.num_experts,
189 + self.hidden_dim,
190 + device=torch.cuda.current_device()))
191 + set_weight_attrs(self.w2, {"weight_loader": self.moe_weight_loader})
192 + self.w3 = nn.Parameter(
193 + torch.empty(self.ffn_dim_per_partition * self.num_experts,
194 + self.hidden_dim,
195 + device=torch.cuda.current_device()))
196 + set_weight_attrs(self.w3, {"weight_loader": self.moe_weight_loader})
197 +
198 + # Calculate the number of bits needed to represent the expert indices
199 + # so that we can pass it to radix sort.
200 + self.sort_end_bit = max(int(np.ceil(np.log2(self.num_experts))), 1)
201 + self.blocking = 128
202 + self.quantize_scatter_num_bits = -1
203 +
204 + # Calculate the number of bits needed to represent the column indices
205 + # in the intermediate sparse matrix.
206 + max_column_index = (self.ffn_dim * self.num_experts) // self.blocking
207 + self.transpose_sort_end_bit = max(
208 + int(np.ceil(np.log2(max_column_index))), 1)
209 +
210 + def moe_weight_loader(self, param: nn.Parameter,
211 + loaded_weight: torch.Tensor) -> None:
212 + """
213 + Load the weights for the MoE linear layer.
214 + """
215 + tp_rank = get_tensor_model_parallel_rank()
216 + shard_size = self.ffn_dim_per_partition
217 + loaded_weight = loaded_weight.view(self.num_experts, self.ffn_dim, -1)
218 + loaded_weight = loaded_weight[:, shard_size * tp_rank:shard_size *
219 + (tp_rank + 1)]
220 + loaded_weight = loaded_weight.reshape_as(param)
221 + param.data.copy_(loaded_weight)
222 +
223 + def sparse_transpose(
224 + self, size: int, row_indices,
225 + column_indices) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
226 + block_columns = size[1] // self.blocking
227 +
228 + # Sort row indices by column indices to get the transposed matrix's
229 + # column indices.
230 + #
231 + # NOTE: Our sort operation uses the same width indices as the input
232 + # values. To avoid overflow when we have large activation matrices
233 + # we cast to 32-bit before sorting.
234 + _, gather_indices = ops.sort(column_indices.int(),
235 + self.transpose_sort_end_bit)
236 +
237 + # There are a constant number of blocks in every row of the sparse
238 + # matrix. A blocks offset is:
239 + #
240 + # row_index * blocks_per_row + column_index % blocks_per_row
241 + #
242 + # Once we have the block offsets ordered for transposition we can
243 + # divide by blocks_per_row to get the transposed column indices.
244 + column_indices_t = row_indices.gather(0, gather_indices.long())
245 + block_offsets_t = gather_indices.int()
246 +
247 + zero = torch.zeros((1, ), dtype=torch.int32, device=row_indices.device)
248 + nnz_per_column = ops.histogram(column_indices, block_columns)
249 + nnz_per_column = ops.inclusive_cumsum(nnz_per_column, 0)
250 + offsets_t = torch.cat([zero, nnz_per_column])
251 + return column_indices_t, offsets_t, block_offsets_t
252 +
253 + def topology(self, x: torch.Tensor,
254 + padded_bins: torch.Tensor) -> stk.Matrix:
255 + padded_tokens, _ = x.size()
256 + assert padded_tokens % self.blocking == 0
257 + assert self.ffn_dim_per_partition % self.blocking == 0
258 +
259 + # Offsets for the sparse matrix. All rows have the
260 + # same number of nonzero blocks dictated by the
261 + # dimensionality of a single expert.
262 + block_rows = padded_tokens // self.blocking
263 + blocks_per_row = self.ffn_dim_per_partition // self.blocking
264 + offsets = torch.arange(
265 + 0,
266 + block_rows * blocks_per_row + 1,
267 + blocks_per_row,
268 + dtype=torch.int32,
269 + device=x.device,
270 + )
271 +
272 + # Indices for the sparse matrix. The indices for
273 + # the intermediate matrix are dynamic depending
274 + # on the mapping of tokens to experts.
275 + column_indices = ops.topology(padded_bins, self.blocking, block_rows,
276 + blocks_per_row)
277 +
278 + # TODO(tgale): This is unused. Remove the need for this in stk.
279 + # For now, use meta init to save the device memory.
280 + data = torch.empty(
281 + column_indices.numel(),
282 + self.blocking,
283 + self.blocking,
284 + dtype=x.dtype,
285 + device="meta",
286 + )
287 + shape = (padded_tokens, self.ffn_dim_per_partition * self.num_experts)
288 + row_indices = stk.ops.row_indices(shape, data, offsets, column_indices)
289 + column_indices_t, offsets_t, block_offsets_t = self.sparse_transpose(
290 + shape, row_indices, column_indices)
291 + return stk.Matrix(
292 + shape,
293 + data,
294 + row_indices,
295 + column_indices,
296 + offsets,
297 + column_indices_t,
298 + offsets_t,
299 + block_offsets_t,
300 + )
301 +
302 + def indices_and_padded_bins(
303 + self, selected_experts: torch.Tensor
304 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
305 + torch.Tensor]:
306 + # Sort the expert ids to produce the scatter/gather
307 + # indices for the permutation.
308 + selected_experts = selected_experts.int()
309 + bin_ids, indices = ops.sort(selected_experts, self.sort_end_bit)
310 +
311 + # Histogram the expert ids to identify the number of
312 + # tokens routed to each expert.
313 + tokens_per_expert = ops.histogram(selected_experts, self.num_experts)
314 +
315 + # Round the token counts up to the block size used in
316 + # the matrix muliplications. Caculate the starting
317 + # position of each bin.
318 + padded_tokens_per_expert = ops.round_up(tokens_per_expert,
319 + self.blocking)
320 + padded_bins = ops.inclusive_cumsum(padded_tokens_per_expert, 0)
321 + padded_bins = promote_scalar(padded_bins)
322 +
323 + # Calculate the bin bounds for the sorted tokens.
324 + bins = ops.inclusive_cumsum(tokens_per_expert, 0)
325 + bins = promote_scalar(bins)
326 + return indices, bin_ids, bins, padded_bins, tokens_per_expert
327 +
328 + @torch.inference_mode()
329 + def forward(self, x: torch.Tensor) -> torch.Tensor:
330 + """
331 + x: (sequence_length, model_dim)
332 + gate_logits: (sequence_length, n_experts)
333 + """
334 + # optional reshape
335 + input_shape = x.shape
336 + x = x.view(-1, input_shape[-1])
337 +
338 + # gate_logits: (sequence_length, n_experts)
339 + gate_logits = self.gate(x)
340 + # all_probs: (sequence_length, n_experts) and upcast for softmax
341 + all_probs = F.softmax(gate_logits, dim=1, dtype=torch.float)
342 + # weights, selected_experts: (sequence_length, top-k)
343 + weights, selected_experts = torch.topk(all_probs, self.top_k, dim=-1)
344 + weights /= weights.sum(dim=-1, keepdim=True)
345 + weights = weights.flatten().to(x.dtype)
346 + selected_experts = selected_experts.flatten()
347 +
348 + (indices, bin_ids, bins, padded_bins,
349 + _) = self.indices_and_padded_bins(selected_experts)
350 +
351 + # Permute tokens and pad to prepare expert computation
352 + # (top_k * sequence_length + padding, model_dim)
353 + x = ops.padded_gather(x, indices, bin_ids, bins, padded_bins,
354 + self.top_k)
355 +
356 + # Create the sparse matrix topology
357 + with torch.no_grad():
358 + topo = self.topology(x, padded_bins)
359 +
360 + # Perform the expert computation
361 + # First Dense x Dense -> Sparse for w1 and w3,
362 + # (top_k * sequence_length + padding, ffn_dim * n_experts)
363 + x = stk.Matrix(
364 + topo.size(),
365 + F.silu(stk.ops.sdd(x, self.w1.t(), topo).data) *
366 + stk.ops.sdd(x, self.w3.t(), topo).data,
367 + topo.row_indices,
368 + topo.column_indices,
369 + topo.offsets,
370 + topo.column_indices_t,
371 + topo.offsets_t,
372 + topo.block_offsets_t,
373 + )
374 +
375 + # Then Sparse x Dense -> Dense for w2
376 + # (top_k * sequence_length + padding, model_dim)
377 + x = stk.ops.dsd(x, self.w2)
378 +
379 + x = tensor_model_parallel_all_reduce(x)
380 +
381 + # Permute back and remove padding
382 + # (top_k * sequence_length, model_dim)
383 + x = ops.padded_scatter(
384 + x,
385 + indices,
386 + bin_ids,
387 + weights,
388 + bins,
389 + padded_bins,
390 + self.top_k,
391 + self.quantize_scatter_num_bits,
392 + )
393 + return x.view(*input_shape)
394 +
395 +
396 +class MixtralDecoderLayer(nn.Module):
397 +
398 + def __init__(
399 + self,
400 + config: MistralConfig,
401 + ) -> None:
402 + super().__init__()
403 + self.hidden_size = config.hidden_size
404 + # Requires transformers > 4.32.0
405 + rope_theta = getattr(config, "rope_theta", 10000)
406 + self.attention = MixtralAttention(
407 + hidden_size=self.hidden_size,
408 + num_heads=config.num_attention_heads,
409 + max_position=config.max_position_embeddings,
410 + num_kv_heads=config.num_key_value_heads,
411 + rope_theta=rope_theta,
412 + sliding_window=config.sliding_window)
413 + self.block_sparse_moe = BlockSparseMoE(
414 + hidden_dim=self.hidden_size,
415 + ffn_dim=config.intermediate_size,
416 + num_experts=config.num_local_experts,
417 + top_k=config.num_experts_per_tok,
418 + )
419 + self.attention_norm = RMSNorm(config.hidden_size,
420 + eps=config.rms_norm_eps)
421 + self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
422 +
423 + def forward(
424 + self,
425 + positions: torch.Tensor,
426 + x: torch.Tensor,
427 + kv_cache: KVCache,
428 + input_metadata: InputMetadata,
429 + cache_event: Optional[torch.cuda.Event],
430 + ) -> torch.Tensor:
431 + r = self.attention(
432 + positions=positions,
433 + hidden_states=self.attention_norm(x),
434 + kv_cache=kv_cache,
435 + input_metadata=input_metadata,
436 + cache_event=cache_event,
437 + )
438 + h = x + r
439 + r = self.block_sparse_moe(self.ffn_norm(h))
440 + out = h + r
441 + return out
442 +
443 +
444 +class MixtralForCausalLM(nn.Module):
445 +
446 + def __init__(
447 + self,
448 + config: MistralConfig,
449 + linear_method: Optional[LinearMethodBase] = None,
450 + ) -> None:
451 + super().__init__()
452 + self.config = config
453 + assert linear_method is None
454 + self.padding_idx = config.pad_token_id
455 + self.vocab_size = config.vocab_size
456 + self.tok_embeddings: Union[nn.Embedding, None] = None
457 + self.layers: nn.ModuleList = None
458 + self.output: Union[nn.Linear, None] = None
459 + self.sampler: Union[Sampler, None] = None
460 + self.tok_embeddings = VocabParallelEmbedding(
461 + config.vocab_size,
462 + config.hidden_size,
463 + )
464 +
465 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
466 + self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
467 + self.sampler = Sampler(config.vocab_size)
468 +
469 + self.layers = nn.ModuleList([
470 + MixtralDecoderLayer(config)
471 + for _ in range(config.num_hidden_layers)
472 + ])
473 +
474 + def forward(
475 + self,
476 + input_ids: torch.Tensor,
477 + positions: torch.Tensor,
478 + kv_caches: List[KVCache],
479 + input_metadata: InputMetadata,
480 + cache_events: Optional[List[torch.cuda.Event]],
481 + ) -> SamplerOutput:
482 + hidden_states = self.tok_embeddings(input_ids)
483 +
484 + # forward
485 + for i in range(len(self.layers)):
486 + cache_event = None if cache_events is None else cache_events[i]
487 + layer = self.layers[i]
488 + hidden_states = layer(
489 + positions,
490 + hidden_states,
491 + kv_caches[i],
492 + input_metadata,
493 + cache_event,
494 + )
495 + return hidden_states
496 +
497 + def sample(
498 + self,
499 + hidden_states: Optional[torch.Tensor],
500 + sampling_metadata: SamplingMetadata,
501 + ) -> SamplerOutput:
502 + hidden_states = self.norm(hidden_states)
503 + next_tokens = self.sampler(self.output.weight, hidden_states,
504 + sampling_metadata)
505 + return next_tokens
506 +
507 + def load_weights(self,
508 + model_name_or_path: str,
509 + cache_dir: Optional[str] = None,
510 + load_format: str = "auto",
511 + revision: Optional[str] = None):
512 + stacked_params_mapping = [
513 + # (param_name, shard_name, shard_id)
514 + ("wqkv", "wq", "q"),
515 + ("wqkv", "wk", "k"),
516 + ("wqkv", "wv", "v"),
517 + ]
518 + params_dict = dict(self.named_parameters())
519 + for name, loaded_weight in hf_model_weights_iterator(
520 + model_name_or_path, cache_dir, load_format, revision):
521 + if "rotary_emb.inv_freq" in name:
522 + continue
523 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
524 + if weight_name not in name:
525 + continue
526 + param = params_dict[name.replace(weight_name, param_name)]
527 + weight_loader = param.weight_loader
528 + weight_loader(param, loaded_weight, shard_id)
529 + break
530 + else:
531 + param = params_dict[name]
532 + weight_loader = getattr(param, "weight_loader",
533 + default_weight_loader)
534 + weight_loader(param, loaded_weight)