A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Bugfix] Fix gpt2 GGUF inference (#12467)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Jan 27, 2025 at 18:31 UTC
ce69f7f7542bdb8b6e6302d112fb9fad212c1460
1 file changed
+8
-11
vllm/model_executor/models/gpt2.py
+8
-11
@@ -258,13 +258,13 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
258
self.transformer = GPT2Model(vllm_config=vllm_config,
259
prefix=maybe_prefix(
260
prefix, "transformer"))
261
+ self.lm_head = ParallelLMHead(self.config.vocab_size,
262
+ self.config.hidden_size,
263
+ quant_config=quant_config,
264
+ prefix=f"{prefix}.lm_head")
265
if self.config.tie_word_embeddings:
262
- self.lm_head = self.transformer.wte
263
- else:
264
- self.lm_head = ParallelLMHead(self.config.vocab_size,
265
- self.config.hidden_size,
266
- quant_config=quant_config,
267
- prefix=f"{prefix}.lm_head")
266
+ self.lm_head = self.lm_head.tie_weights(self.transformer.wte)
267
+
268
self.logits_processor = LogitsProcessor(config.vocab_size)
269
self.sampler = get_sampler()
270
self.make_empty_intermediate_tensors = (
@@ -309,15 +309,12 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
309
params_dict = dict(self.named_parameters(remove_duplicate=False))
310
loaded_params: Set[str] = set()
311
for name, loaded_weight in weights:
312
- if name.startswith("lm_head"):
313
- # GPT-2 ties the weights of the embedding layer and the final
314
- # linear layer.
315
- continue
312
if ".attn.bias" in name or ".attn.masked_bias" in name:
313
# Skip attention mask.
314
# NOTE: "c_attn.bias" should not be skipped.
315
continue
320
- if not name.startswith("transformer."):
316
+ if not name.startswith("transformer.") and not name.startswith(
317
+ "lm_head"):
318
name = "transformer." + name
319
320
if is_pp_missing_parameter(name, self):