@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ce69f7f754
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Bugfix] Fix gpt2 GGUF inference (#12467)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Jan 27, 2025 at 18:31 UTC ce69f7f7542bdb8b6e6302d112fb9fad212c1460
1 file changed +8 -11
vllm/model_executor/models/gpt2.py
+8 -11
@@ -258,13 +258,13 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
258 self.transformer = GPT2Model(vllm_config=vllm_config,
259 prefix=maybe_prefix(
260 prefix, "transformer"))
261 + self.lm_head = ParallelLMHead(self.config.vocab_size,
262 + self.config.hidden_size,
263 + quant_config=quant_config,
264 + prefix=f"{prefix}.lm_head")
265 if self.config.tie_word_embeddings:
262 - self.lm_head = self.transformer.wte
263 - else:
264 - self.lm_head = ParallelLMHead(self.config.vocab_size,
265 - self.config.hidden_size,
266 - quant_config=quant_config,
267 - prefix=f"{prefix}.lm_head")
266 + self.lm_head = self.lm_head.tie_weights(self.transformer.wte)
267 +
268 self.logits_processor = LogitsProcessor(config.vocab_size)
269 self.sampler = get_sampler()
270 self.make_empty_intermediate_tensors = (
@@ -309,15 +309,12 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
309 params_dict = dict(self.named_parameters(remove_duplicate=False))
310 loaded_params: Set[str] = set()
311 for name, loaded_weight in weights:
312 - if name.startswith("lm_head"):
313 - # GPT-2 ties the weights of the embedding layer and the final
314 - # linear layer.
315 - continue
312 if ".attn.bias" in name or ".attn.masked_bias" in name:
313 # Skip attention mask.
314 # NOTE: "c_attn.bias" should not be skipped.
315 continue
320 - if not name.startswith("transformer."):
316 + if not name.startswith("transformer.") and not name.startswith(
317 + "lm_head"):
318 name = "transformer." + name
319
320 if is_pp_missing_parameter(name, self):