A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Mixtral 8x7B support (#2011)
Co-authored-by: Pierre Stock <p@mistral.ai> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
Pierre Stock committed
Dec 11, 2023 at 10:09 UTC
b5f882cc98e2c9c6dde7357dbac2ec0c2c57d8cd
4 files changed
+538
README.md
+1
@@ -60,6 +60,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
60
- InternLM (`internlm/internlm-7b`, `internlm/internlm-chat-7b`, etc.)
61
- LLaMA & LLaMA-2 (`meta-llama/Llama-2-70b-hf`, `lmsys/vicuna-13b-v1.3`, `young-geng/koala`, `openlm-research/open_llama_13b`, etc.)
62
- Mistral (`mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc.)
63
+- Mixtral (`mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, etc.)
64
- MPT (`mosaicml/mpt-7b`, `mosaicml/mpt-30b`, etc.)
65
- OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
66
- Phi-1.5 (`microsoft/phi-1_5`, etc.)
vllm/model_executor/model_loader.py
+1
@@ -33,6 +33,7 @@ _MODEL_REGISTRY = {
33
"LlamaForCausalLM": LlamaForCausalLM,
34
"LLaMAForCausalLM": LlamaForCausalLM, # For decapoda-research/llama-*
35
"MistralForCausalLM": MistralForCausalLM,
36
+ "MixtralForCausalLM": MixtralForCausalLM,
37
# transformers's mpt class has lower case
38
"MptForCausalLM": MPTForCausalLM,
39
"MPTForCausalLM": MPTForCausalLM,
vllm/model_executor/models/__init__.py
+2
@@ -10,6 +10,7 @@ from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
10
from vllm.model_executor.models.internlm import InternLMForCausalLM
11
from vllm.model_executor.models.llama import LlamaForCausalLM
12
from vllm.model_executor.models.mistral import MistralForCausalLM
13
+from vllm.model_executor.models.mixtral import MixtralForCausalLM
14
from vllm.model_executor.models.mpt import MPTForCausalLM
15
from vllm.model_executor.models.opt import OPTForCausalLM
16
from vllm.model_executor.models.phi_1_5 import PhiForCausalLM
@@ -35,5 +36,6 @@ __all__ = [
36
"PhiForCausalLM",
37
"QWenLMHeadModel",
38
"MistralForCausalLM",
39
+ "MixtralForCausalLM",
40
"YiForCausalLM",
41
]
vllm/model_executor/models/mixtral.py
new
+534
@@ -0,0 +1,534 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only Mixtral model."""
24
+from typing import List, Optional, Tuple, Union
25
+
26
+import numpy as np
27
+
28
+import torch
29
+import torch.nn.functional as F
30
+
31
+from torch import nn
32
+from transformers import MistralConfig
33
+
34
+try:
35
+ import megablocks.ops as ops
36
+except ImportError:
37
+ print("MegaBlocks not found, please see "
38
+ "https://github.com/stanford-futuredata/megablocks/. "
39
+ "Note that MegaBlocks depends on mosaicml-turbo, which only "
40
+ "supports python 3.10.")
41
+try:
42
+ import stk
43
+except ImportError:
44
+ print(
45
+ "STK not found: please see https://github.com/stanford-futuredata/stk")
46
+
47
+from vllm.model_executor.input_metadata import InputMetadata
48
+from vllm.model_executor.layers.attention import PagedAttention
49
+from vllm.model_executor.layers.layernorm import RMSNorm
50
+from vllm.model_executor.layers.linear import (LinearMethodBase,
51
+ QKVParallelLinear,
52
+ RowParallelLinear)
53
+from vllm.model_executor.layers.rotary_embedding import get_rope
54
+from vllm.model_executor.layers.sampler import Sampler
55
+from vllm.model_executor.layers.vocab_parallel_embedding import (
56
+ VocabParallelEmbedding, ParallelLMHead)
57
+from vllm.model_executor.parallel_utils.communication_op import (
58
+ tensor_model_parallel_all_reduce)
59
+from vllm.model_executor.parallel_utils.parallel_state import (
60
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
61
+from vllm.model_executor.sampling_metadata import SamplingMetadata
62
+from vllm.model_executor.weight_utils import (default_weight_loader,
63
+ hf_model_weights_iterator)
64
+from vllm.model_executor.utils import set_weight_attrs
65
+from vllm.sequence import SamplerOutput
66
+
67
+KVCache = Tuple[torch.Tensor, torch.Tensor]
68
+
69
+
70
+def promote_scalar(x: torch.Tensor) -> torch.Tensor:
71
+ return x.view(1) if len(x.size()) == 0 else x
72
+
73
+
74
+class MixtralAttention(nn.Module):
75
+
76
+ def __init__(self,
77
+ hidden_size: int,
78
+ num_heads: int,
79
+ num_kv_heads: int,
80
+ max_position: int = 4096 * 32,
81
+ rope_theta: float = 10000,
82
+ sliding_window: Optional[int] = None) -> None:
83
+ super().__init__()
84
+ self.hidden_size = hidden_size
85
+ tp_size = get_tensor_model_parallel_world_size()
86
+ self.total_num_heads = num_heads
87
+ assert self.total_num_heads % tp_size == 0
88
+ self.num_heads = self.total_num_heads // tp_size
89
+ self.total_num_kv_heads = num_kv_heads
90
+ if self.total_num_kv_heads >= tp_size:
91
+ # Number of KV heads is greater than TP size, so we partition
92
+ # the KV heads across multiple tensor parallel GPUs.
93
+ assert self.total_num_kv_heads % tp_size == 0
94
+ else:
95
+ # Number of KV heads is less than TP size, so we replicate
96
+ # the KV heads across multiple tensor parallel GPUs.
97
+ assert tp_size % self.total_num_kv_heads == 0
98
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
99
+ self.head_dim = hidden_size // self.total_num_heads
100
+ self.q_size = self.num_heads * self.head_dim
101
+ self.kv_size = self.num_kv_heads * self.head_dim
102
+ self.scaling = self.head_dim**-0.5
103
+ self.rope_theta = rope_theta
104
+ self.sliding_window = sliding_window
105
+
106
+ self.wqkv = QKVParallelLinear(
107
+ hidden_size,
108
+ self.head_dim,
109
+ self.total_num_heads,
110
+ self.total_num_kv_heads,
111
+ bias=False,
112
+ )
113
+ self.wo = RowParallelLinear(
114
+ self.total_num_heads * self.head_dim,
115
+ hidden_size,
116
+ bias=False,
117
+ )
118
+ self.rotary_emb = get_rope(
119
+ self.head_dim,
120
+ rotary_dim=self.head_dim,
121
+ max_position=max_position,
122
+ base=int(self.rope_theta),
123
+ is_neox_style=False, # weights not in HF format
124
+ )
125
+ self.attn = PagedAttention(
126
+ self.num_heads,
127
+ self.head_dim,
128
+ self.scaling,
129
+ num_kv_heads=self.num_kv_heads,
130
+ sliding_window=self.sliding_window,
131
+ )
132
+
133
+ def forward(
134
+ self,
135
+ positions: torch.Tensor,
136
+ hidden_states: torch.Tensor,
137
+ kv_cache: KVCache,
138
+ input_metadata: InputMetadata,
139
+ cache_event: Optional[torch.cuda.Event],
140
+ ) -> torch.Tensor:
141
+ qkv, _ = self.wqkv(hidden_states)
142
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
143
+ q, k = self.rotary_emb(positions, q, k)
144
+ k_cache, v_cache = kv_cache
145
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
146
+ cache_event)
147
+ output, _ = self.wo(attn_output)
148
+ return output
149
+
150
+
151
+class BlockSparseMoE(nn.Module):
152
+ """
153
+ Built on the paper and library Megablocks as described in
154
+ https://arxiv.org/abs/2211.15841. This implementation is
155
+ strictly equivalent to standard MoE with full capacity (no
156
+ dropped tokens). It's faster since it formulates MoE operations
157
+ in terms of block-sparse operations to accomodate imbalanced
158
+ assignments of tokens to experts, whereas standard MoE either
159
+ (1) drop tokens at the cost of reduced performance or (2) set
160
+ capacity factor to number of experts and thus waste computation
161
+ and memory on padding.
162
+ """
163
+
164
+ def __init__(self, hidden_dim: int, ffn_dim: int, num_experts: int,
165
+ top_k: int):
166
+ super().__init__()
167
+ self.hidden_dim = hidden_dim
168
+ self.ffn_dim = ffn_dim
169
+ self.num_experts = num_experts
170
+ self.top_k = top_k
171
+
172
+ # gating
173
+ self.gate = nn.Linear(self.hidden_dim,
174
+ self.num_experts,
175
+ bias=False,
176
+ device=torch.cuda.current_device())
177
+
178
+ tp_size = get_tensor_model_parallel_world_size()
179
+ assert self.ffn_dim % tp_size == 0
180
+ self.ffn_dim_per_partition = self.ffn_dim // tp_size
181
+ # merged expert weights, all of size (ffn_dim * n_experts, model_dim)
182
+ self.w1 = nn.Parameter(
183
+ torch.empty(self.ffn_dim_per_partition * self.num_experts,
184
+ self.hidden_dim,
185
+ device=torch.cuda.current_device()))
186
+ set_weight_attrs(self.w1, {"weight_loader": self.moe_weight_loader})
187
+ self.w2 = nn.Parameter(
188
+ torch.empty(self.ffn_dim_per_partition * self.num_experts,
189
+ self.hidden_dim,
190
+ device=torch.cuda.current_device()))
191
+ set_weight_attrs(self.w2, {"weight_loader": self.moe_weight_loader})
192
+ self.w3 = nn.Parameter(
193
+ torch.empty(self.ffn_dim_per_partition * self.num_experts,
194
+ self.hidden_dim,
195
+ device=torch.cuda.current_device()))
196
+ set_weight_attrs(self.w3, {"weight_loader": self.moe_weight_loader})
197
+
198
+ # Calculate the number of bits needed to represent the expert indices
199
+ # so that we can pass it to radix sort.
200
+ self.sort_end_bit = max(int(np.ceil(np.log2(self.num_experts))), 1)
201
+ self.blocking = 128
202
+ self.quantize_scatter_num_bits = -1
203
+
204
+ # Calculate the number of bits needed to represent the column indices
205
+ # in the intermediate sparse matrix.
206
+ max_column_index = (self.ffn_dim * self.num_experts) // self.blocking
207
+ self.transpose_sort_end_bit = max(
208
+ int(np.ceil(np.log2(max_column_index))), 1)
209
+
210
+ def moe_weight_loader(self, param: nn.Parameter,
211
+ loaded_weight: torch.Tensor) -> None:
212
+ """
213
+ Load the weights for the MoE linear layer.
214
+ """
215
+ tp_rank = get_tensor_model_parallel_rank()
216
+ shard_size = self.ffn_dim_per_partition
217
+ loaded_weight = loaded_weight.view(self.num_experts, self.ffn_dim, -1)
218
+ loaded_weight = loaded_weight[:, shard_size * tp_rank:shard_size *
219
+ (tp_rank + 1)]
220
+ loaded_weight = loaded_weight.reshape_as(param)
221
+ param.data.copy_(loaded_weight)
222
+
223
+ def sparse_transpose(
224
+ self, size: int, row_indices,
225
+ column_indices) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
226
+ block_columns = size[1] // self.blocking
227
+
228
+ # Sort row indices by column indices to get the transposed matrix's
229
+ # column indices.
230
+ #
231
+ # NOTE: Our sort operation uses the same width indices as the input
232
+ # values. To avoid overflow when we have large activation matrices
233
+ # we cast to 32-bit before sorting.
234
+ _, gather_indices = ops.sort(column_indices.int(),
235
+ self.transpose_sort_end_bit)
236
+
237
+ # There are a constant number of blocks in every row of the sparse
238
+ # matrix. A blocks offset is:
239
+ #
240
+ # row_index * blocks_per_row + column_index % blocks_per_row
241
+ #
242
+ # Once we have the block offsets ordered for transposition we can
243
+ # divide by blocks_per_row to get the transposed column indices.
244
+ column_indices_t = row_indices.gather(0, gather_indices.long())
245
+ block_offsets_t = gather_indices.int()
246
+
247
+ zero = torch.zeros((1, ), dtype=torch.int32, device=row_indices.device)
248
+ nnz_per_column = ops.histogram(column_indices, block_columns)
249
+ nnz_per_column = ops.inclusive_cumsum(nnz_per_column, 0)
250
+ offsets_t = torch.cat([zero, nnz_per_column])
251
+ return column_indices_t, offsets_t, block_offsets_t
252
+
253
+ def topology(self, x: torch.Tensor,
254
+ padded_bins: torch.Tensor) -> stk.Matrix:
255
+ padded_tokens, _ = x.size()
256
+ assert padded_tokens % self.blocking == 0
257
+ assert self.ffn_dim_per_partition % self.blocking == 0
258
+
259
+ # Offsets for the sparse matrix. All rows have the
260
+ # same number of nonzero blocks dictated by the
261
+ # dimensionality of a single expert.
262
+ block_rows = padded_tokens // self.blocking
263
+ blocks_per_row = self.ffn_dim_per_partition // self.blocking
264
+ offsets = torch.arange(
265
+ 0,
266
+ block_rows * blocks_per_row + 1,
267
+ blocks_per_row,
268
+ dtype=torch.int32,
269
+ device=x.device,
270
+ )
271
+
272
+ # Indices for the sparse matrix. The indices for
273
+ # the intermediate matrix are dynamic depending
274
+ # on the mapping of tokens to experts.
275
+ column_indices = ops.topology(padded_bins, self.blocking, block_rows,
276
+ blocks_per_row)
277
+
278
+ # TODO(tgale): This is unused. Remove the need for this in stk.
279
+ # For now, use meta init to save the device memory.
280
+ data = torch.empty(
281
+ column_indices.numel(),
282
+ self.blocking,
283
+ self.blocking,
284
+ dtype=x.dtype,
285
+ device="meta",
286
+ )
287
+ shape = (padded_tokens, self.ffn_dim_per_partition * self.num_experts)
288
+ row_indices = stk.ops.row_indices(shape, data, offsets, column_indices)
289
+ column_indices_t, offsets_t, block_offsets_t = self.sparse_transpose(
290
+ shape, row_indices, column_indices)
291
+ return stk.Matrix(
292
+ shape,
293
+ data,
294
+ row_indices,
295
+ column_indices,
296
+ offsets,
297
+ column_indices_t,
298
+ offsets_t,
299
+ block_offsets_t,
300
+ )
301
+
302
+ def indices_and_padded_bins(
303
+ self, selected_experts: torch.Tensor
304
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
305
+ torch.Tensor]:
306
+ # Sort the expert ids to produce the scatter/gather
307
+ # indices for the permutation.
308
+ selected_experts = selected_experts.int()
309
+ bin_ids, indices = ops.sort(selected_experts, self.sort_end_bit)
310
+
311
+ # Histogram the expert ids to identify the number of
312
+ # tokens routed to each expert.
313
+ tokens_per_expert = ops.histogram(selected_experts, self.num_experts)
314
+
315
+ # Round the token counts up to the block size used in
316
+ # the matrix muliplications. Caculate the starting
317
+ # position of each bin.
318
+ padded_tokens_per_expert = ops.round_up(tokens_per_expert,
319
+ self.blocking)
320
+ padded_bins = ops.inclusive_cumsum(padded_tokens_per_expert, 0)
321
+ padded_bins = promote_scalar(padded_bins)
322
+
323
+ # Calculate the bin bounds for the sorted tokens.
324
+ bins = ops.inclusive_cumsum(tokens_per_expert, 0)
325
+ bins = promote_scalar(bins)
326
+ return indices, bin_ids, bins, padded_bins, tokens_per_expert
327
+
328
+ @torch.inference_mode()
329
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
330
+ """
331
+ x: (sequence_length, model_dim)
332
+ gate_logits: (sequence_length, n_experts)
333
+ """
334
+ # optional reshape
335
+ input_shape = x.shape
336
+ x = x.view(-1, input_shape[-1])
337
+
338
+ # gate_logits: (sequence_length, n_experts)
339
+ gate_logits = self.gate(x)
340
+ # all_probs: (sequence_length, n_experts) and upcast for softmax
341
+ all_probs = F.softmax(gate_logits, dim=1, dtype=torch.float)
342
+ # weights, selected_experts: (sequence_length, top-k)
343
+ weights, selected_experts = torch.topk(all_probs, self.top_k, dim=-1)
344
+ weights /= weights.sum(dim=-1, keepdim=True)
345
+ weights = weights.flatten().to(x.dtype)
346
+ selected_experts = selected_experts.flatten()
347
+
348
+ (indices, bin_ids, bins, padded_bins,
349
+ _) = self.indices_and_padded_bins(selected_experts)
350
+
351
+ # Permute tokens and pad to prepare expert computation
352
+ # (top_k * sequence_length + padding, model_dim)
353
+ x = ops.padded_gather(x, indices, bin_ids, bins, padded_bins,
354
+ self.top_k)
355
+
356
+ # Create the sparse matrix topology
357
+ with torch.no_grad():
358
+ topo = self.topology(x, padded_bins)
359
+
360
+ # Perform the expert computation
361
+ # First Dense x Dense -> Sparse for w1 and w3,
362
+ # (top_k * sequence_length + padding, ffn_dim * n_experts)
363
+ x = stk.Matrix(
364
+ topo.size(),
365
+ F.silu(stk.ops.sdd(x, self.w1.t(), topo).data) *
366
+ stk.ops.sdd(x, self.w3.t(), topo).data,
367
+ topo.row_indices,
368
+ topo.column_indices,
369
+ topo.offsets,
370
+ topo.column_indices_t,
371
+ topo.offsets_t,
372
+ topo.block_offsets_t,
373
+ )
374
+
375
+ # Then Sparse x Dense -> Dense for w2
376
+ # (top_k * sequence_length + padding, model_dim)
377
+ x = stk.ops.dsd(x, self.w2)
378
+
379
+ x = tensor_model_parallel_all_reduce(x)
380
+
381
+ # Permute back and remove padding
382
+ # (top_k * sequence_length, model_dim)
383
+ x = ops.padded_scatter(
384
+ x,
385
+ indices,
386
+ bin_ids,
387
+ weights,
388
+ bins,
389
+ padded_bins,
390
+ self.top_k,
391
+ self.quantize_scatter_num_bits,
392
+ )
393
+ return x.view(*input_shape)
394
+
395
+
396
+class MixtralDecoderLayer(nn.Module):
397
+
398
+ def __init__(
399
+ self,
400
+ config: MistralConfig,
401
+ ) -> None:
402
+ super().__init__()
403
+ self.hidden_size = config.hidden_size
404
+ # Requires transformers > 4.32.0
405
+ rope_theta = getattr(config, "rope_theta", 10000)
406
+ self.attention = MixtralAttention(
407
+ hidden_size=self.hidden_size,
408
+ num_heads=config.num_attention_heads,
409
+ max_position=config.max_position_embeddings,
410
+ num_kv_heads=config.num_key_value_heads,
411
+ rope_theta=rope_theta,
412
+ sliding_window=config.sliding_window)
413
+ self.block_sparse_moe = BlockSparseMoE(
414
+ hidden_dim=self.hidden_size,
415
+ ffn_dim=config.intermediate_size,
416
+ num_experts=config.num_local_experts,
417
+ top_k=config.num_experts_per_tok,
418
+ )
419
+ self.attention_norm = RMSNorm(config.hidden_size,
420
+ eps=config.rms_norm_eps)
421
+ self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
422
+
423
+ def forward(
424
+ self,
425
+ positions: torch.Tensor,
426
+ x: torch.Tensor,
427
+ kv_cache: KVCache,
428
+ input_metadata: InputMetadata,
429
+ cache_event: Optional[torch.cuda.Event],
430
+ ) -> torch.Tensor:
431
+ r = self.attention(
432
+ positions=positions,
433
+ hidden_states=self.attention_norm(x),
434
+ kv_cache=kv_cache,
435
+ input_metadata=input_metadata,
436
+ cache_event=cache_event,
437
+ )
438
+ h = x + r
439
+ r = self.block_sparse_moe(self.ffn_norm(h))
440
+ out = h + r
441
+ return out
442
+
443
+
444
+class MixtralForCausalLM(nn.Module):
445
+
446
+ def __init__(
447
+ self,
448
+ config: MistralConfig,
449
+ linear_method: Optional[LinearMethodBase] = None,
450
+ ) -> None:
451
+ super().__init__()
452
+ self.config = config
453
+ assert linear_method is None
454
+ self.padding_idx = config.pad_token_id
455
+ self.vocab_size = config.vocab_size
456
+ self.tok_embeddings: Union[nn.Embedding, None] = None
457
+ self.layers: nn.ModuleList = None
458
+ self.output: Union[nn.Linear, None] = None
459
+ self.sampler: Union[Sampler, None] = None
460
+ self.tok_embeddings = VocabParallelEmbedding(
461
+ config.vocab_size,
462
+ config.hidden_size,
463
+ )
464
+
465
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
466
+ self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
467
+ self.sampler = Sampler(config.vocab_size)
468
+
469
+ self.layers = nn.ModuleList([
470
+ MixtralDecoderLayer(config)
471
+ for _ in range(config.num_hidden_layers)
472
+ ])
473
+
474
+ def forward(
475
+ self,
476
+ input_ids: torch.Tensor,
477
+ positions: torch.Tensor,
478
+ kv_caches: List[KVCache],
479
+ input_metadata: InputMetadata,
480
+ cache_events: Optional[List[torch.cuda.Event]],
481
+ ) -> SamplerOutput:
482
+ hidden_states = self.tok_embeddings(input_ids)
483
+
484
+ # forward
485
+ for i in range(len(self.layers)):
486
+ cache_event = None if cache_events is None else cache_events[i]
487
+ layer = self.layers[i]
488
+ hidden_states = layer(
489
+ positions,
490
+ hidden_states,
491
+ kv_caches[i],
492
+ input_metadata,
493
+ cache_event,
494
+ )
495
+ return hidden_states
496
+
497
+ def sample(
498
+ self,
499
+ hidden_states: Optional[torch.Tensor],
500
+ sampling_metadata: SamplingMetadata,
501
+ ) -> SamplerOutput:
502
+ hidden_states = self.norm(hidden_states)
503
+ next_tokens = self.sampler(self.output.weight, hidden_states,
504
+ sampling_metadata)
505
+ return next_tokens
506
+
507
+ def load_weights(self,
508
+ model_name_or_path: str,
509
+ cache_dir: Optional[str] = None,
510
+ load_format: str = "auto",
511
+ revision: Optional[str] = None):
512
+ stacked_params_mapping = [
513
+ # (param_name, shard_name, shard_id)
514
+ ("wqkv", "wq", "q"),
515
+ ("wqkv", "wk", "k"),
516
+ ("wqkv", "wv", "v"),
517
+ ]
518
+ params_dict = dict(self.named_parameters())
519
+ for name, loaded_weight in hf_model_weights_iterator(
520
+ model_name_or_path, cache_dir, load_format, revision):
521
+ if "rotary_emb.inv_freq" in name:
522
+ continue
523
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
524
+ if weight_name not in name:
525
+ continue
526
+ param = params_dict[name.replace(weight_name, param_name)]
527
+ weight_loader = param.weight_loader
528
+ weight_loader(param, loaded_weight, shard_id)
529
+ break
530
+ else:
531
+ param = params_dict[name]
532
+ weight_loader = getattr(param, "weight_loader",
533
+ default_weight_loader)
534
+ weight_loader(param, loaded_weight)