A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
expand coverage of gpt2 model loading (#271)
twaka committed
Jun 27, 2023 at 22:27 UTC
4026a049d3ad510bea8e177bf71722bd510fbb46
1 file changed
+4
-2
vllm/model_executor/models/gpt2.py
+4
-2
@@ -228,11 +228,13 @@ class GPT2LMHeadModel(nn.Module):
228
# GPT-2 ties the weights of the embedding layer and the final
229
# linear layer.
230
continue
231
- if ".attn.bias" in name:
231
+ if ".attn.bias" in name or ".attn.masked_bias" in name:
232
# Skip attention mask.
233
# NOTE: "c_attn.bias" should not be skipped.
234
continue
235
- name = "transformer." + name
235
+
236
+ if not name.startswith("transformer."):
237
+ name = "transformer." + name
238
239
# The HF's GPT-2 implementation uses Conv1D instead of Linear.
240
# Because of this, we need to transpose the weights.