A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Model] Add Gemma 2 (#5908)
Woosuk Kwon committed
Jun 27, 2024 at 13:33 UTC
79c92c7c8aa6a881421e2007ab216a819f61bc9b
9 files changed
+499
-9
docs/source/models/supported_models.rst
+4
@@ -55,6 +55,10 @@ Alongside each architecture, we include some popular models that use it.
55
- Gemma
56
- :code:`google/gemma-2b`, :code:`google/gemma-7b`, etc.
57
- ✅︎
58
+ * - :code:`Gemma2ForCausalLM`
59
+ - Gemma2
60
+ - :code:`google/gemma-2-9b`, :code:`google/gemma-2-27b`, etc.
61
+ - ✅︎
62
* - :code:`GPT2LMHeadModel`
63
- GPT-2
64
- :code:`gpt2`, :code:`gpt2-xl`, etc.
requirements-common.txt
+1
-1
@@ -6,7 +6,7 @@ numpy < 2.0.0
6
requests
7
tqdm
8
py-cpuinfo
9
-transformers >= 4.40.0 # Required for StarCoder2 & Llava, Llama 3.
9
+transformers >= 4.42.0 # Required for Gemma 2.
10
tokenizers >= 0.19.1 # Required for Llama 3.
11
fastapi
12
aiohttp
vllm/config.py
+23
-7
@@ -14,7 +14,7 @@ from vllm.model_executor.models import ModelRegistry
14
from vllm.tracing import is_otel_installed
15
from vllm.transformers_utils.config import get_config, get_hf_text_config
16
from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
17
- is_hip, is_neuron, is_tpu, is_xpu,
17
+ is_hip, is_neuron, is_tpu, is_xpu, print_warning_once,
18
update_environment_variables)
19
20
if TYPE_CHECKING:
@@ -141,6 +141,17 @@ class ModelConfig:
141
code_revision, rope_scaling, rope_theta)
142
self.hf_text_config = get_hf_text_config(self.hf_config)
143
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
144
+
145
+ if (not self.disable_sliding_window
146
+ and self.hf_text_config.model_type == "gemma2"
147
+ and self.hf_text_config.sliding_window is not None):
148
+ print_warning_once(
149
+ "Gemma 2 uses sliding window attention for every odd layer, "
150
+ "which is currently not supported by vLLM. Disabling sliding "
151
+ "window and capping the max length to the sliding window size "
152
+ f"({self.hf_text_config.sliding_window}).")
153
+ self.disable_sliding_window = True
154
+
155
self.max_model_len = _get_and_verify_max_len(
156
hf_config=self.hf_text_config,
157
max_model_len=max_model_len,
@@ -257,8 +268,7 @@ class ModelConfig:
268
"BitAndBytes quantization with TP or PP is not supported yet.")
269
270
def get_hf_config_sliding_window(self) -> Optional[int]:
260
- """Get the sliding window size, or None if disabled.
261
- """
271
+ """Get the sliding window size, or None if disabled."""
272
273
# Some models, like Qwen2 and Qwen1.5, use `use_sliding_window` in
274
# addition to sliding window size. We check if that field is present
@@ -1256,10 +1266,16 @@ def _get_and_verify_dtype(
1266
dtype = dtype.lower()
1267
if dtype == "auto":
1268
if config_dtype == torch.float32:
1259
- # Following the common practice, we use float16 for float32
1260
- # models.
1261
- logger.info("Casting torch.float32 to torch.float16.")
1262
- torch_dtype = torch.float16
1269
+ if config.model_type == "gemma2":
1270
+ logger.info(
1271
+ "For Gemma 2, we downcast float32 to bfloat16 instead "
1272
+ "of float16 by default. Please specify `dtype` if you "
1273
+ "want to use float16.")
1274
+ torch_dtype = torch.bfloat16
1275
+ else:
1276
+ # Following the common practice, we use float16 for float32
1277
+ # models.
1278
+ torch_dtype = torch.float16
1279
else:
1280
torch_dtype = config_dtype
1281
else:
vllm/lora/layers.py
+4
@@ -1069,6 +1069,10 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1069
def scale(self):
1070
return self.base_layer.scale
1071
1072
+ @property
1073
+ def soft_cap(self):
1074
+ return self.base_layer.soft_cap
1075
+
1076
@property
1077
def org_vocab_size(self):
1078
return self.base_layer.org_vocab_size
vllm/model_executor/layers/layernorm.py
+46
@@ -95,3 +95,49 @@ class RMSNorm(CustomOp):
95
s = f"hidden_size={self.weight.data.size(0)}"
96
s += f", eps={self.variance_epsilon}"
97
return s
98
+
99
+
100
+class GemmaRMSNorm(CustomOp):
101
+ """RMS normalization for Gemma.
102
+
103
+ Two differences from the above RMSNorm:
104
+ 1. x * (1 + w) instead of x * w.
105
+ 2. (x * w).to(orig_dtype) instead of x.to(orig_dtype) * w.
106
+ """
107
+
108
+ def __init__(
109
+ self,
110
+ hidden_size: int,
111
+ eps: float = 1e-6,
112
+ ) -> None:
113
+ super().__init__()
114
+ self.weight = nn.Parameter(torch.zeros(hidden_size))
115
+ self.variance_epsilon = eps
116
+
117
+ def forward_native(
118
+ self,
119
+ x: torch.Tensor,
120
+ residual: Optional[torch.Tensor] = None,
121
+ ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
122
+ """PyTorch-native implementation equivalent to forward()."""
123
+ orig_dtype = x.dtype
124
+ if residual is not None:
125
+ x = x + residual
126
+ residual = x
127
+
128
+ x = x.float()
129
+ variance = x.pow(2).mean(dim=-1, keepdim=True)
130
+ x = x * torch.rsqrt(variance + self.variance_epsilon)
131
+ # Llama does x.to(float16) * w whilst Gemma is (x * w).to(float16)
132
+ # See https://github.com/huggingface/transformers/pull/29402
133
+ x = x * (1.0 + self.weight.float())
134
+ x = x.to(orig_dtype)
135
+ return x if residual is None else (x, residual)
136
+
137
+ def forward_cuda(
138
+ self,
139
+ x: torch.Tensor,
140
+ residual: Optional[torch.Tensor] = None,
141
+ ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
142
+ # TODO(woosuk): Implement an optimized kernel for GemmaRMSNorm.
143
+ return self.forward_native(x, residual)
vllm/model_executor/layers/logits_processor.py
+9
-1
@@ -22,7 +22,8 @@ class LogitsProcessor(nn.Module):
22
vocab_size: int,
23
org_vocab_size: Optional[int] = None,
24
scale: float = 1.0,
25
- logits_as_input: bool = False) -> None:
25
+ logits_as_input: bool = False,
26
+ soft_cap: Optional[float] = None) -> None:
27
"""
28
Args:
29
scale: A scaling factor to apply to the logits.
@@ -34,6 +35,8 @@ class LogitsProcessor(nn.Module):
35
self.logits_as_input = logits_as_input
36
# original vocabulary size (without LoRA).
37
self.org_vocab_size = org_vocab_size or vocab_size
38
+ # Soft cap the logits. Used in Gemma 2.
39
+ self.soft_cap = soft_cap
40
41
def forward(
42
self,
@@ -52,6 +55,11 @@ class LogitsProcessor(nn.Module):
55
logits = self._get_logits(hidden_states, embedding, embedding_bias)
56
57
if logits is not None:
58
+ if self.soft_cap is not None:
59
+ logits = logits / self.soft_cap
60
+ logits = torch.tanh(logits)
61
+ logits = logits * self.soft_cap
62
+
63
if self.scale != 1.0:
64
logits *= self.scale
65
vllm/model_executor/layers/rotary_embedding.py
+10
@@ -610,6 +610,16 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
610
return query.flatten(-2), key.flatten(-2)
611
612
613
+class GemmaRotaryEmbedding(RotaryEmbedding):
614
+
615
+ def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
616
+ # https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/gemma/modeling_gemma.py#L107
617
+ inv_freq = 1.0 / (base**(
618
+ torch.arange(0, self.rotary_dim, 2, dtype=torch.int64).float() /
619
+ self.rotary_dim))
620
+ return inv_freq
621
+
622
+
623
_ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
624
625
vllm/model_executor/models/__init__.py
+1
@@ -23,6 +23,7 @@ _GENERATION_MODELS = {
23
"DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
24
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
25
"GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
26
+ "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
27
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
28
"GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
29
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
vllm/model_executor/models/gemma2.py
new
+401
@@ -0,0 +1,401 @@
1
+# coding=utf-8
2
+# Copyright 2024 The vLLM team.
3
+# Copyright 2024 Google Inc. HuggingFace Inc. team. All rights reserved.
4
+#
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+from typing import Iterable, List, Optional, Set, Tuple
18
+
19
+import torch
20
+from torch import nn
21
+from transformers import Gemma2Config
22
+
23
+from vllm.attention import Attention, AttentionMetadata
24
+from vllm.config import CacheConfig, LoRAConfig
25
+from vllm.distributed import get_tensor_model_parallel_world_size
26
+from vllm.model_executor.layers.activation import GeluAndMul
27
+from vllm.model_executor.layers.layernorm import GemmaRMSNorm
28
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
29
+ QKVParallelLinear,
30
+ RowParallelLinear)
31
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
32
+from vllm.model_executor.layers.quantization.base_config import (
33
+ QuantizationConfig)
34
+from vllm.model_executor.layers.rotary_embedding import GemmaRotaryEmbedding
35
+from vllm.model_executor.layers.sampler import Sampler
36
+from vllm.model_executor.layers.vocab_parallel_embedding import (
37
+ VocabParallelEmbedding)
38
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39
+from vllm.model_executor.sampling_metadata import SamplingMetadata
40
+from vllm.sequence import SamplerOutput
41
+from vllm.utils import print_warning_once
42
+
43
+from .interfaces import SupportsLoRA
44
+
45
+
46
+class Gemma2MLP(nn.Module):
47
+
48
+ def __init__(
49
+ self,
50
+ hidden_size: int,
51
+ intermediate_size: int,
52
+ hidden_act: str,
53
+ hidden_activation: str,
54
+ quant_config: Optional[QuantizationConfig] = None,
55
+ ) -> None:
56
+ super().__init__()
57
+ self.gate_up_proj = MergedColumnParallelLinear(
58
+ hidden_size, [intermediate_size] * 2,
59
+ bias=False,
60
+ quant_config=quant_config)
61
+ self.down_proj = RowParallelLinear(intermediate_size,
62
+ hidden_size,
63
+ bias=False,
64
+ quant_config=quant_config)
65
+ if not (hidden_act == hidden_activation == "gelu_pytorch_tanh"):
66
+ raise ValueError(
67
+ "Gemma2 uses `gelu_pytorch_tanh` as the hidden activation "
68
+ "function. Please set `hidden_act` and `hidden_activation` to "
69
+ "`gelu_pytorch_tanh`.")
70
+ self.act_fn = GeluAndMul(approximate="tanh")
71
+
72
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
73
+ gate_up, _ = self.gate_up_proj(x)
74
+ x = self.act_fn(gate_up)
75
+ x, _ = self.down_proj(x)
76
+ return x
77
+
78
+
79
+class Gemma2Attention(nn.Module):
80
+
81
+ def __init__(self,
82
+ layer_idx: int,
83
+ config: Gemma2Config,
84
+ hidden_size: int,
85
+ num_heads: int,
86
+ num_kv_heads: int,
87
+ head_dim: int,
88
+ max_position_embeddings: int,
89
+ rope_theta: float,
90
+ cache_config: Optional[CacheConfig] = None,
91
+ quant_config: Optional[QuantizationConfig] = None) -> None:
92
+ super().__init__()
93
+ self.layer_idx = layer_idx
94
+ self.config = config
95
+ self.hidden_size = hidden_size
96
+ tp_size = get_tensor_model_parallel_world_size()
97
+ self.total_num_heads = num_heads
98
+ assert self.total_num_heads % tp_size == 0
99
+ self.num_heads = self.total_num_heads // tp_size
100
+ self.total_num_kv_heads = num_kv_heads
101
+ if self.total_num_kv_heads >= tp_size:
102
+ # Number of KV heads is greater than TP size, so we partition
103
+ # the KV heads across multiple tensor parallel GPUs.
104
+ assert self.total_num_kv_heads % tp_size == 0
105
+ else:
106
+ # Number of KV heads is less than TP size, so we replicate
107
+ # the KV heads across multiple tensor parallel GPUs.
108
+ assert tp_size % self.total_num_kv_heads == 0
109
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
110
+ self.head_dim = head_dim
111
+ self.q_size = self.num_heads * self.head_dim
112
+ self.kv_size = self.num_kv_heads * self.head_dim
113
+ self.scaling = config.query_pre_attn_scalar**-0.5
114
+ self.rope_theta = rope_theta
115
+
116
+ self.qkv_proj = QKVParallelLinear(
117
+ hidden_size,
118
+ self.head_dim,
119
+ self.total_num_heads,
120
+ self.total_num_kv_heads,
121
+ bias=config.attention_bias,
122
+ quant_config=quant_config,
123
+ )
124
+ self.o_proj = RowParallelLinear(
125
+ self.total_num_heads * self.head_dim,
126
+ hidden_size,
127
+ bias=config.attention_bias,
128
+ quant_config=quant_config,
129
+ )
130
+ # TODO(woosuk): Use the `get_rope` interface.
131
+ self.rotary_emb = GemmaRotaryEmbedding(
132
+ self.head_dim,
133
+ self.head_dim,
134
+ max_position_embeddings,
135
+ base=self.rope_theta,
136
+ is_neox_style=True,
137
+ dtype=torch.get_default_dtype(),
138
+ )
139
+
140
+ if self.config.attn_logit_softcapping is not None:
141
+ print_warning_once(
142
+ "Gemma 2 normally uses attention logit soft-capping; "
143
+ "soft-capping is currently incompatible with the flash "
144
+ "attention kernels, so vLLM removes it to enable speed and "
145
+ "efficiency gains of flash attention.")
146
+ # FIXME(woosuk): While Gemma 2 uses sliding window attention for every
147
+ # odd layer, vLLM currently ignores it and uses global attention for
148
+ # all layers.
149
+ use_sliding_window = (layer_idx % 2 == 1
150
+ and config.sliding_window is not None)
151
+ del use_sliding_window # Unused.
152
+ self.attn = Attention(self.num_heads,
153
+ self.head_dim,
154
+ self.scaling,
155
+ num_kv_heads=self.num_kv_heads,
156
+ cache_config=cache_config,
157
+ quant_config=quant_config)
158
+
159
+ def forward(
160
+ self,
161
+ positions: torch.Tensor,
162
+ hidden_states: torch.Tensor,
163
+ kv_cache: torch.Tensor,
164
+ attn_metadata: AttentionMetadata,
165
+ ) -> torch.Tensor:
166
+ qkv, _ = self.qkv_proj(hidden_states)
167
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
168
+ q, k = self.rotary_emb(positions, q, k)
169
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
170
+ output, _ = self.o_proj(attn_output)
171
+ return output
172
+
173
+
174
+class Gemma2DecoderLayer(nn.Module):
175
+
176
+ def __init__(
177
+ self,
178
+ layer_idx: int,
179
+ config: Gemma2Config,
180
+ cache_config: Optional[CacheConfig] = None,
181
+ quant_config: Optional[QuantizationConfig] = None,
182
+ ) -> None:
183
+ super().__init__()
184
+ self.hidden_size = config.hidden_size
185
+ self.self_attn = Gemma2Attention(
186
+ layer_idx=layer_idx,
187
+ config=config,
188
+ hidden_size=self.hidden_size,
189
+ num_heads=config.num_attention_heads,
190
+ num_kv_heads=config.num_key_value_heads,
191
+ head_dim=config.head_dim,
192
+ max_position_embeddings=config.max_position_embeddings,
193
+ rope_theta=config.rope_theta,
194
+ cache_config=cache_config,
195
+ quant_config=quant_config,
196
+ )
197
+ self.hidden_size = config.hidden_size
198
+ self.mlp = Gemma2MLP(
199
+ hidden_size=self.hidden_size,
200
+ intermediate_size=config.intermediate_size,
201
+ hidden_act=config.hidden_act,
202
+ hidden_activation=config.hidden_activation,
203
+ quant_config=quant_config,
204
+ )
205
+ self.input_layernorm = GemmaRMSNorm(config.hidden_size,
206
+ eps=config.rms_norm_eps)
207
+ self.post_attention_layernorm = GemmaRMSNorm(config.hidden_size,
208
+ eps=config.rms_norm_eps)
209
+ self.pre_feedforward_layernorm = GemmaRMSNorm(config.hidden_size,
210
+ eps=config.rms_norm_eps)
211
+ self.post_feedforward_layernorm = GemmaRMSNorm(config.hidden_size,
212
+ eps=config.rms_norm_eps)
213
+
214
+ def forward(
215
+ self,
216
+ positions: torch.Tensor,
217
+ hidden_states: torch.Tensor,
218
+ kv_cache: torch.Tensor,
219
+ attn_metadata: AttentionMetadata,
220
+ residual: Optional[torch.Tensor],
221
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
222
+ if residual is None:
223
+ residual = hidden_states
224
+ hidden_states = self.input_layernorm(hidden_states)
225
+ else:
226
+ hidden_states, residual = self.input_layernorm(
227
+ hidden_states, residual)
228
+ hidden_states = self.self_attn(
229
+ positions=positions,
230
+ hidden_states=hidden_states,
231
+ kv_cache=kv_cache,
232
+ attn_metadata=attn_metadata,
233
+ )
234
+ hidden_states = self.post_attention_layernorm(hidden_states)
235
+
236
+ hidden_states, residual = self.pre_feedforward_layernorm(
237
+ hidden_states, residual)
238
+ hidden_states = self.mlp(hidden_states)
239
+ hidden_states = self.post_feedforward_layernorm(hidden_states)
240
+ return hidden_states, residual
241
+
242
+
243
+class Gemma2Model(nn.Module):
244
+
245
+ def __init__(
246
+ self,
247
+ config: Gemma2Config,
248
+ cache_config: Optional[CacheConfig] = None,
249
+ quant_config: Optional[QuantizationConfig] = None,
250
+ ) -> None:
251
+ super().__init__()
252
+ self.config = config
253
+
254
+ self.embed_tokens = VocabParallelEmbedding(
255
+ config.vocab_size,
256
+ config.hidden_size,
257
+ )
258
+ self.layers = nn.ModuleList([
259
+ Gemma2DecoderLayer(layer_idx, config, cache_config, quant_config)
260
+ for layer_idx in range(config.num_hidden_layers)
261
+ ])
262
+ self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
263
+
264
+ # Normalize the embedding by sqrt(hidden_size)
265
+ # The normalizer's data type should be downcasted to the model's
266
+ # data type such as bfloat16, not float32.
267
+ # See https://github.com/huggingface/transformers/pull/29402
268
+ normalizer = self.config.hidden_size**0.5
269
+ self.register_buffer("normalizer", torch.tensor(normalizer))
270
+
271
+ def forward(
272
+ self,
273
+ input_ids: torch.Tensor,
274
+ positions: torch.Tensor,
275
+ kv_caches: List[torch.Tensor],
276
+ attn_metadata: AttentionMetadata,
277
+ ) -> torch.Tensor:
278
+ hidden_states = self.embed_tokens(input_ids)
279
+ hidden_states *= self.normalizer
280
+
281
+ residual = None
282
+ for i in range(len(self.layers)):
283
+ layer = self.layers[i]
284
+ hidden_states, residual = layer(
285
+ positions,
286
+ hidden_states,
287
+ kv_caches[i],
288
+ attn_metadata,
289
+ residual,
290
+ )
291
+ hidden_states, _ = self.norm(hidden_states, residual)
292
+ return hidden_states
293
+
294
+
295
+class Gemma2ForCausalLM(nn.Module, SupportsLoRA):
296
+ packed_modules_mapping = {
297
+ "qkv_proj": [
298
+ "q_proj",
299
+ "k_proj",
300
+ "v_proj",
301
+ ],
302
+ "gate_up_proj": [
303
+ "gate_proj",
304
+ "up_proj",
305
+ ],
306
+ }
307
+
308
+ # LoRA specific attributes
309
+ supported_lora_modules = [
310
+ "qkv_proj",
311
+ "o_proj",
312
+ "gate_up_proj",
313
+ "down_proj",
314
+ ]
315
+ # Gemma does not apply LoRA to the embedding layer.
316
+ embedding_modules = {}
317
+ embedding_padding_modules = []
318
+
319
+ def __init__(
320
+ self,
321
+ config: Gemma2Config,
322
+ cache_config: Optional[CacheConfig] = None,
323
+ quant_config: Optional[QuantizationConfig] = None,
324
+ lora_config: Optional[LoRAConfig] = None,
325
+ ) -> None:
326
+ del lora_config # Unused.
327
+ super().__init__()
328
+ self.config = config
329
+ self.quant_config = quant_config
330
+ self.model = Gemma2Model(config, cache_config, quant_config)
331
+ self.logits_processor = LogitsProcessor(
332
+ config.vocab_size, soft_cap=config.final_logit_softcapping)
333
+ self.sampler = Sampler()
334
+
335
+ def forward(
336
+ self,
337
+ input_ids: torch.Tensor,
338
+ positions: torch.Tensor,
339
+ kv_caches: List[torch.Tensor],
340
+ attn_metadata: AttentionMetadata,
341
+ ) -> torch.Tensor:
342
+ hidden_states = self.model(input_ids, positions, kv_caches,
343
+ attn_metadata)
344
+ return hidden_states
345
+
346
+ def compute_logits(self, hidden_states: torch.Tensor,
347
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
348
+ logits = self.logits_processor(self.model.embed_tokens.weight,
349
+ hidden_states, sampling_metadata)
350
+ return logits
351
+
352
+ def sample(
353
+ self,
354
+ logits: torch.Tensor,
355
+ sampling_metadata: SamplingMetadata,
356
+ ) -> Optional[SamplerOutput]:
357
+ next_tokens = self.sampler(logits, sampling_metadata)
358
+ return next_tokens
359
+
360
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
361
+ stacked_params_mapping = [
362
+ # (param_name, shard_name, shard_id)
363
+ ("qkv_proj", "q_proj", "q"),
364
+ ("qkv_proj", "k_proj", "k"),
365
+ ("qkv_proj", "v_proj", "v"),
366
+ ("gate_up_proj", "gate_proj", 0),
367
+ ("gate_up_proj", "up_proj", 1),
368
+ ]
369
+ params_dict = dict(self.named_parameters())
370
+ loaded_params: Set[str] = set()
371
+ for name, loaded_weight in weights:
372
+ for (param_name, shard_name, shard_id) in stacked_params_mapping:
373
+ if shard_name not in name:
374
+ continue
375
+ name = name.replace(shard_name, param_name)
376
+ # Skip loading extra bias for GPTQ models.
377
+ if name.endswith(".bias") and name not in params_dict:
378
+ continue
379
+ param = params_dict[name]
380
+ weight_loader = param.weight_loader
381
+ weight_loader(param, loaded_weight, shard_id)
382
+ break
383
+ else:
384
+ # lm_head is not used in vllm as it is tied with embed_token.
385
+ # To prevent errors, skip loading lm_head.weight.
386
+ if "lm_head.weight" in name:
387
+ continue
388
+ # Skip loading extra bias for GPTQ models.
389
+ if name.endswith(".bias") and name not in params_dict:
390
+ continue
391
+ param = params_dict[name]
392
+ weight_loader = getattr(param, "weight_loader",
393
+ default_weight_loader)
394
+ weight_loader(param, loaded_weight)
395
+ loaded_params.add(name)
396
+
397
+ unloaded_params = params_dict.keys() - loaded_params
398
+ if unloaded_params:
399
+ raise RuntimeError(
400
+ "Some weights are not initialized from checkpoints: "
401
+ f"{unloaded_params}")