@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 79c92c7c8a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Model] Add Gemma 2 (#5908)

Woosuk Kwon committed Jun 27, 2024 at 13:33 UTC 79c92c7c8aa6a881421e2007ab216a819f61bc9b
9 files changed +499 -9
docs/source/models/supported_models.rst
+4
@@ -55,6 +55,10 @@ Alongside each architecture, we include some popular models that use it.
55 - Gemma
56 - :code:`google/gemma-2b`, :code:`google/gemma-7b`, etc.
57 - ✅︎
58 + * - :code:`Gemma2ForCausalLM`
59 + - Gemma2
60 + - :code:`google/gemma-2-9b`, :code:`google/gemma-2-27b`, etc.
61 + - ✅︎
62 * - :code:`GPT2LMHeadModel`
63 - GPT-2
64 - :code:`gpt2`, :code:`gpt2-xl`, etc.
requirements-common.txt
+1 -1
@@ -6,7 +6,7 @@ numpy < 2.0.0
6 requests
7 tqdm
8 py-cpuinfo
9 -transformers >= 4.40.0 # Required for StarCoder2 & Llava, Llama 3.
9 +transformers >= 4.42.0 # Required for Gemma 2.
10 tokenizers >= 0.19.1 # Required for Llama 3.
11 fastapi
12 aiohttp
vllm/config.py
+23 -7
@@ -14,7 +14,7 @@ from vllm.model_executor.models import ModelRegistry
14 from vllm.tracing import is_otel_installed
15 from vllm.transformers_utils.config import get_config, get_hf_text_config
16 from vllm.utils import (cuda_device_count_stateless, get_cpu_memory, is_cpu,
17 - is_hip, is_neuron, is_tpu, is_xpu,
17 + is_hip, is_neuron, is_tpu, is_xpu, print_warning_once,
18 update_environment_variables)
19
20 if TYPE_CHECKING:
@@ -141,6 +141,17 @@ class ModelConfig:
141 code_revision, rope_scaling, rope_theta)
142 self.hf_text_config = get_hf_text_config(self.hf_config)
143 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
144 +
145 + if (not self.disable_sliding_window
146 + and self.hf_text_config.model_type == "gemma2"
147 + and self.hf_text_config.sliding_window is not None):
148 + print_warning_once(
149 + "Gemma 2 uses sliding window attention for every odd layer, "
150 + "which is currently not supported by vLLM. Disabling sliding "
151 + "window and capping the max length to the sliding window size "
152 + f"({self.hf_text_config.sliding_window}).")
153 + self.disable_sliding_window = True
154 +
155 self.max_model_len = _get_and_verify_max_len(
156 hf_config=self.hf_text_config,
157 max_model_len=max_model_len,
@@ -257,8 +268,7 @@ class ModelConfig:
268 "BitAndBytes quantization with TP or PP is not supported yet.")
269
270 def get_hf_config_sliding_window(self) -> Optional[int]:
260 - """Get the sliding window size, or None if disabled.
261 - """
271 + """Get the sliding window size, or None if disabled."""
272
273 # Some models, like Qwen2 and Qwen1.5, use `use_sliding_window` in
274 # addition to sliding window size. We check if that field is present
@@ -1256,10 +1266,16 @@ def _get_and_verify_dtype(
1266 dtype = dtype.lower()
1267 if dtype == "auto":
1268 if config_dtype == torch.float32:
1259 - # Following the common practice, we use float16 for float32
1260 - # models.
1261 - logger.info("Casting torch.float32 to torch.float16.")
1262 - torch_dtype = torch.float16
1269 + if config.model_type == "gemma2":
1270 + logger.info(
1271 + "For Gemma 2, we downcast float32 to bfloat16 instead "
1272 + "of float16 by default. Please specify `dtype` if you "
1273 + "want to use float16.")
1274 + torch_dtype = torch.bfloat16
1275 + else:
1276 + # Following the common practice, we use float16 for float32
1277 + # models.
1278 + torch_dtype = torch.float16
1279 else:
1280 torch_dtype = config_dtype
1281 else:
vllm/lora/layers.py
+4
@@ -1069,6 +1069,10 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1069 def scale(self):
1070 return self.base_layer.scale
1071
1072 + @property
1073 + def soft_cap(self):
1074 + return self.base_layer.soft_cap
1075 +
1076 @property
1077 def org_vocab_size(self):
1078 return self.base_layer.org_vocab_size
vllm/model_executor/layers/layernorm.py
+46
@@ -95,3 +95,49 @@ class RMSNorm(CustomOp):
95 s = f"hidden_size={self.weight.data.size(0)}"
96 s += f", eps={self.variance_epsilon}"
97 return s
98 +
99 +
100 +class GemmaRMSNorm(CustomOp):
101 + """RMS normalization for Gemma.
102 +
103 + Two differences from the above RMSNorm:
104 + 1. x * (1 + w) instead of x * w.
105 + 2. (x * w).to(orig_dtype) instead of x.to(orig_dtype) * w.
106 + """
107 +
108 + def __init__(
109 + self,
110 + hidden_size: int,
111 + eps: float = 1e-6,
112 + ) -> None:
113 + super().__init__()
114 + self.weight = nn.Parameter(torch.zeros(hidden_size))
115 + self.variance_epsilon = eps
116 +
117 + def forward_native(
118 + self,
119 + x: torch.Tensor,
120 + residual: Optional[torch.Tensor] = None,
121 + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
122 + """PyTorch-native implementation equivalent to forward()."""
123 + orig_dtype = x.dtype
124 + if residual is not None:
125 + x = x + residual
126 + residual = x
127 +
128 + x = x.float()
129 + variance = x.pow(2).mean(dim=-1, keepdim=True)
130 + x = x * torch.rsqrt(variance + self.variance_epsilon)
131 + # Llama does x.to(float16) * w whilst Gemma is (x * w).to(float16)
132 + # See https://github.com/huggingface/transformers/pull/29402
133 + x = x * (1.0 + self.weight.float())
134 + x = x.to(orig_dtype)
135 + return x if residual is None else (x, residual)
136 +
137 + def forward_cuda(
138 + self,
139 + x: torch.Tensor,
140 + residual: Optional[torch.Tensor] = None,
141 + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
142 + # TODO(woosuk): Implement an optimized kernel for GemmaRMSNorm.
143 + return self.forward_native(x, residual)
vllm/model_executor/layers/logits_processor.py
+9 -1
@@ -22,7 +22,8 @@ class LogitsProcessor(nn.Module):
22 vocab_size: int,
23 org_vocab_size: Optional[int] = None,
24 scale: float = 1.0,
25 - logits_as_input: bool = False) -> None:
25 + logits_as_input: bool = False,
26 + soft_cap: Optional[float] = None) -> None:
27 """
28 Args:
29 scale: A scaling factor to apply to the logits.
@@ -34,6 +35,8 @@ class LogitsProcessor(nn.Module):
35 self.logits_as_input = logits_as_input
36 # original vocabulary size (without LoRA).
37 self.org_vocab_size = org_vocab_size or vocab_size
38 + # Soft cap the logits. Used in Gemma 2.
39 + self.soft_cap = soft_cap
40
41 def forward(
42 self,
@@ -52,6 +55,11 @@ class LogitsProcessor(nn.Module):
55 logits = self._get_logits(hidden_states, embedding, embedding_bias)
56
57 if logits is not None:
58 + if self.soft_cap is not None:
59 + logits = logits / self.soft_cap
60 + logits = torch.tanh(logits)
61 + logits = logits * self.soft_cap
62 +
63 if self.scale != 1.0:
64 logits *= self.scale
65
vllm/model_executor/layers/rotary_embedding.py
+10
@@ -610,6 +610,16 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
610 return query.flatten(-2), key.flatten(-2)
611
612
613 +class GemmaRotaryEmbedding(RotaryEmbedding):
614 +
615 + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
616 + # https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/gemma/modeling_gemma.py#L107
617 + inv_freq = 1.0 / (base**(
618 + torch.arange(0, self.rotary_dim, 2, dtype=torch.int64).float() /
619 + self.rotary_dim))
620 + return inv_freq
621 +
622 +
623 _ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
624
625
vllm/model_executor/models/__init__.py
+1
@@ -23,6 +23,7 @@ _GENERATION_MODELS = {
23 "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
24 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
25 "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
26 + "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
27 "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
28 "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
29 "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
vllm/model_executor/models/gemma2.py new
+401
@@ -0,0 +1,401 @@
1 +# coding=utf-8
2 +# Copyright 2024 The vLLM team.
3 +# Copyright 2024 Google Inc. HuggingFace Inc. team. All rights reserved.
4 +#
5 +#
6 +# Licensed under the Apache License, Version 2.0 (the "License");
7 +# you may not use this file except in compliance with the License.
8 +# You may obtain a copy of the License at
9 +#
10 +# http://www.apache.org/licenses/LICENSE-2.0
11 +#
12 +# Unless required by applicable law or agreed to in writing, software
13 +# distributed under the License is distributed on an "AS IS" BASIS,
14 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 +# See the License for the specific language governing permissions and
16 +# limitations under the License.
17 +from typing import Iterable, List, Optional, Set, Tuple
18 +
19 +import torch
20 +from torch import nn
21 +from transformers import Gemma2Config
22 +
23 +from vllm.attention import Attention, AttentionMetadata
24 +from vllm.config import CacheConfig, LoRAConfig
25 +from vllm.distributed import get_tensor_model_parallel_world_size
26 +from vllm.model_executor.layers.activation import GeluAndMul
27 +from vllm.model_executor.layers.layernorm import GemmaRMSNorm
28 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
29 + QKVParallelLinear,
30 + RowParallelLinear)
31 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
32 +from vllm.model_executor.layers.quantization.base_config import (
33 + QuantizationConfig)
34 +from vllm.model_executor.layers.rotary_embedding import GemmaRotaryEmbedding
35 +from vllm.model_executor.layers.sampler import Sampler
36 +from vllm.model_executor.layers.vocab_parallel_embedding import (
37 + VocabParallelEmbedding)
38 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39 +from vllm.model_executor.sampling_metadata import SamplingMetadata
40 +from vllm.sequence import SamplerOutput
41 +from vllm.utils import print_warning_once
42 +
43 +from .interfaces import SupportsLoRA
44 +
45 +
46 +class Gemma2MLP(nn.Module):
47 +
48 + def __init__(
49 + self,
50 + hidden_size: int,
51 + intermediate_size: int,
52 + hidden_act: str,
53 + hidden_activation: str,
54 + quant_config: Optional[QuantizationConfig] = None,
55 + ) -> None:
56 + super().__init__()
57 + self.gate_up_proj = MergedColumnParallelLinear(
58 + hidden_size, [intermediate_size] * 2,
59 + bias=False,
60 + quant_config=quant_config)
61 + self.down_proj = RowParallelLinear(intermediate_size,
62 + hidden_size,
63 + bias=False,
64 + quant_config=quant_config)
65 + if not (hidden_act == hidden_activation == "gelu_pytorch_tanh"):
66 + raise ValueError(
67 + "Gemma2 uses `gelu_pytorch_tanh` as the hidden activation "
68 + "function. Please set `hidden_act` and `hidden_activation` to "
69 + "`gelu_pytorch_tanh`.")
70 + self.act_fn = GeluAndMul(approximate="tanh")
71 +
72 + def forward(self, x: torch.Tensor) -> torch.Tensor:
73 + gate_up, _ = self.gate_up_proj(x)
74 + x = self.act_fn(gate_up)
75 + x, _ = self.down_proj(x)
76 + return x
77 +
78 +
79 +class Gemma2Attention(nn.Module):
80 +
81 + def __init__(self,
82 + layer_idx: int,
83 + config: Gemma2Config,
84 + hidden_size: int,
85 + num_heads: int,
86 + num_kv_heads: int,
87 + head_dim: int,
88 + max_position_embeddings: int,
89 + rope_theta: float,
90 + cache_config: Optional[CacheConfig] = None,
91 + quant_config: Optional[QuantizationConfig] = None) -> None:
92 + super().__init__()
93 + self.layer_idx = layer_idx
94 + self.config = config
95 + self.hidden_size = hidden_size
96 + tp_size = get_tensor_model_parallel_world_size()
97 + self.total_num_heads = num_heads
98 + assert self.total_num_heads % tp_size == 0
99 + self.num_heads = self.total_num_heads // tp_size
100 + self.total_num_kv_heads = num_kv_heads
101 + if self.total_num_kv_heads >= tp_size:
102 + # Number of KV heads is greater than TP size, so we partition
103 + # the KV heads across multiple tensor parallel GPUs.
104 + assert self.total_num_kv_heads % tp_size == 0
105 + else:
106 + # Number of KV heads is less than TP size, so we replicate
107 + # the KV heads across multiple tensor parallel GPUs.
108 + assert tp_size % self.total_num_kv_heads == 0
109 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
110 + self.head_dim = head_dim
111 + self.q_size = self.num_heads * self.head_dim
112 + self.kv_size = self.num_kv_heads * self.head_dim
113 + self.scaling = config.query_pre_attn_scalar**-0.5
114 + self.rope_theta = rope_theta
115 +
116 + self.qkv_proj = QKVParallelLinear(
117 + hidden_size,
118 + self.head_dim,
119 + self.total_num_heads,
120 + self.total_num_kv_heads,
121 + bias=config.attention_bias,
122 + quant_config=quant_config,
123 + )
124 + self.o_proj = RowParallelLinear(
125 + self.total_num_heads * self.head_dim,
126 + hidden_size,
127 + bias=config.attention_bias,
128 + quant_config=quant_config,
129 + )
130 + # TODO(woosuk): Use the `get_rope` interface.
131 + self.rotary_emb = GemmaRotaryEmbedding(
132 + self.head_dim,
133 + self.head_dim,
134 + max_position_embeddings,
135 + base=self.rope_theta,
136 + is_neox_style=True,
137 + dtype=torch.get_default_dtype(),
138 + )
139 +
140 + if self.config.attn_logit_softcapping is not None:
141 + print_warning_once(
142 + "Gemma 2 normally uses attention logit soft-capping; "
143 + "soft-capping is currently incompatible with the flash "
144 + "attention kernels, so vLLM removes it to enable speed and "
145 + "efficiency gains of flash attention.")
146 + # FIXME(woosuk): While Gemma 2 uses sliding window attention for every
147 + # odd layer, vLLM currently ignores it and uses global attention for
148 + # all layers.
149 + use_sliding_window = (layer_idx % 2 == 1
150 + and config.sliding_window is not None)
151 + del use_sliding_window # Unused.
152 + self.attn = Attention(self.num_heads,
153 + self.head_dim,
154 + self.scaling,
155 + num_kv_heads=self.num_kv_heads,
156 + cache_config=cache_config,
157 + quant_config=quant_config)
158 +
159 + def forward(
160 + self,
161 + positions: torch.Tensor,
162 + hidden_states: torch.Tensor,
163 + kv_cache: torch.Tensor,
164 + attn_metadata: AttentionMetadata,
165 + ) -> torch.Tensor:
166 + qkv, _ = self.qkv_proj(hidden_states)
167 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
168 + q, k = self.rotary_emb(positions, q, k)
169 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
170 + output, _ = self.o_proj(attn_output)
171 + return output
172 +
173 +
174 +class Gemma2DecoderLayer(nn.Module):
175 +
176 + def __init__(
177 + self,
178 + layer_idx: int,
179 + config: Gemma2Config,
180 + cache_config: Optional[CacheConfig] = None,
181 + quant_config: Optional[QuantizationConfig] = None,
182 + ) -> None:
183 + super().__init__()
184 + self.hidden_size = config.hidden_size
185 + self.self_attn = Gemma2Attention(
186 + layer_idx=layer_idx,
187 + config=config,
188 + hidden_size=self.hidden_size,
189 + num_heads=config.num_attention_heads,
190 + num_kv_heads=config.num_key_value_heads,
191 + head_dim=config.head_dim,
192 + max_position_embeddings=config.max_position_embeddings,
193 + rope_theta=config.rope_theta,
194 + cache_config=cache_config,
195 + quant_config=quant_config,
196 + )
197 + self.hidden_size = config.hidden_size
198 + self.mlp = Gemma2MLP(
199 + hidden_size=self.hidden_size,
200 + intermediate_size=config.intermediate_size,
201 + hidden_act=config.hidden_act,
202 + hidden_activation=config.hidden_activation,
203 + quant_config=quant_config,
204 + )
205 + self.input_layernorm = GemmaRMSNorm(config.hidden_size,
206 + eps=config.rms_norm_eps)
207 + self.post_attention_layernorm = GemmaRMSNorm(config.hidden_size,
208 + eps=config.rms_norm_eps)
209 + self.pre_feedforward_layernorm = GemmaRMSNorm(config.hidden_size,
210 + eps=config.rms_norm_eps)
211 + self.post_feedforward_layernorm = GemmaRMSNorm(config.hidden_size,
212 + eps=config.rms_norm_eps)
213 +
214 + def forward(
215 + self,
216 + positions: torch.Tensor,
217 + hidden_states: torch.Tensor,
218 + kv_cache: torch.Tensor,
219 + attn_metadata: AttentionMetadata,
220 + residual: Optional[torch.Tensor],
221 + ) -> Tuple[torch.Tensor, torch.Tensor]:
222 + if residual is None:
223 + residual = hidden_states
224 + hidden_states = self.input_layernorm(hidden_states)
225 + else:
226 + hidden_states, residual = self.input_layernorm(
227 + hidden_states, residual)
228 + hidden_states = self.self_attn(
229 + positions=positions,
230 + hidden_states=hidden_states,
231 + kv_cache=kv_cache,
232 + attn_metadata=attn_metadata,
233 + )
234 + hidden_states = self.post_attention_layernorm(hidden_states)
235 +
236 + hidden_states, residual = self.pre_feedforward_layernorm(
237 + hidden_states, residual)
238 + hidden_states = self.mlp(hidden_states)
239 + hidden_states = self.post_feedforward_layernorm(hidden_states)
240 + return hidden_states, residual
241 +
242 +
243 +class Gemma2Model(nn.Module):
244 +
245 + def __init__(
246 + self,
247 + config: Gemma2Config,
248 + cache_config: Optional[CacheConfig] = None,
249 + quant_config: Optional[QuantizationConfig] = None,
250 + ) -> None:
251 + super().__init__()
252 + self.config = config
253 +
254 + self.embed_tokens = VocabParallelEmbedding(
255 + config.vocab_size,
256 + config.hidden_size,
257 + )
258 + self.layers = nn.ModuleList([
259 + Gemma2DecoderLayer(layer_idx, config, cache_config, quant_config)
260 + for layer_idx in range(config.num_hidden_layers)
261 + ])
262 + self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
263 +
264 + # Normalize the embedding by sqrt(hidden_size)
265 + # The normalizer's data type should be downcasted to the model's
266 + # data type such as bfloat16, not float32.
267 + # See https://github.com/huggingface/transformers/pull/29402
268 + normalizer = self.config.hidden_size**0.5
269 + self.register_buffer("normalizer", torch.tensor(normalizer))
270 +
271 + def forward(
272 + self,
273 + input_ids: torch.Tensor,
274 + positions: torch.Tensor,
275 + kv_caches: List[torch.Tensor],
276 + attn_metadata: AttentionMetadata,
277 + ) -> torch.Tensor:
278 + hidden_states = self.embed_tokens(input_ids)
279 + hidden_states *= self.normalizer
280 +
281 + residual = None
282 + for i in range(len(self.layers)):
283 + layer = self.layers[i]
284 + hidden_states, residual = layer(
285 + positions,
286 + hidden_states,
287 + kv_caches[i],
288 + attn_metadata,
289 + residual,
290 + )
291 + hidden_states, _ = self.norm(hidden_states, residual)
292 + return hidden_states
293 +
294 +
295 +class Gemma2ForCausalLM(nn.Module, SupportsLoRA):
296 + packed_modules_mapping = {
297 + "qkv_proj": [
298 + "q_proj",
299 + "k_proj",
300 + "v_proj",
301 + ],
302 + "gate_up_proj": [
303 + "gate_proj",
304 + "up_proj",
305 + ],
306 + }
307 +
308 + # LoRA specific attributes
309 + supported_lora_modules = [
310 + "qkv_proj",
311 + "o_proj",
312 + "gate_up_proj",
313 + "down_proj",
314 + ]
315 + # Gemma does not apply LoRA to the embedding layer.
316 + embedding_modules = {}
317 + embedding_padding_modules = []
318 +
319 + def __init__(
320 + self,
321 + config: Gemma2Config,
322 + cache_config: Optional[CacheConfig] = None,
323 + quant_config: Optional[QuantizationConfig] = None,
324 + lora_config: Optional[LoRAConfig] = None,
325 + ) -> None:
326 + del lora_config # Unused.
327 + super().__init__()
328 + self.config = config
329 + self.quant_config = quant_config
330 + self.model = Gemma2Model(config, cache_config, quant_config)
331 + self.logits_processor = LogitsProcessor(
332 + config.vocab_size, soft_cap=config.final_logit_softcapping)
333 + self.sampler = Sampler()
334 +
335 + def forward(
336 + self,
337 + input_ids: torch.Tensor,
338 + positions: torch.Tensor,
339 + kv_caches: List[torch.Tensor],
340 + attn_metadata: AttentionMetadata,
341 + ) -> torch.Tensor:
342 + hidden_states = self.model(input_ids, positions, kv_caches,
343 + attn_metadata)
344 + return hidden_states
345 +
346 + def compute_logits(self, hidden_states: torch.Tensor,
347 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
348 + logits = self.logits_processor(self.model.embed_tokens.weight,
349 + hidden_states, sampling_metadata)
350 + return logits
351 +
352 + def sample(
353 + self,
354 + logits: torch.Tensor,
355 + sampling_metadata: SamplingMetadata,
356 + ) -> Optional[SamplerOutput]:
357 + next_tokens = self.sampler(logits, sampling_metadata)
358 + return next_tokens
359 +
360 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
361 + stacked_params_mapping = [
362 + # (param_name, shard_name, shard_id)
363 + ("qkv_proj", "q_proj", "q"),
364 + ("qkv_proj", "k_proj", "k"),
365 + ("qkv_proj", "v_proj", "v"),
366 + ("gate_up_proj", "gate_proj", 0),
367 + ("gate_up_proj", "up_proj", 1),
368 + ]
369 + params_dict = dict(self.named_parameters())
370 + loaded_params: Set[str] = set()
371 + for name, loaded_weight in weights:
372 + for (param_name, shard_name, shard_id) in stacked_params_mapping:
373 + if shard_name not in name:
374 + continue
375 + name = name.replace(shard_name, param_name)
376 + # Skip loading extra bias for GPTQ models.
377 + if name.endswith(".bias") and name not in params_dict:
378 + continue
379 + param = params_dict[name]
380 + weight_loader = param.weight_loader
381 + weight_loader(param, loaded_weight, shard_id)
382 + break
383 + else:
384 + # lm_head is not used in vllm as it is tied with embed_token.
385 + # To prevent errors, skip loading lm_head.weight.
386 + if "lm_head.weight" in name:
387 + continue
388 + # Skip loading extra bias for GPTQ models.
389 + if name.endswith(".bias") and name not in params_dict:
390 + continue
391 + param = params_dict[name]
392 + weight_loader = getattr(param, "weight_loader",
393 + default_weight_loader)
394 + weight_loader(param, loaded_weight)
395 + loaded_params.add(name)
396 +
397 + unloaded_params = params_dict.keys() - loaded_params
398 + if unloaded_params:
399 + raise RuntimeError(
400 + "Some weights are not initialized from checkpoints: "
401 + f"{unloaded_params}")