@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e23a43aef8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Fix KeyError on loading GPT-NeoX (#3925)

Junichi Sato committed Apr 10, 2024 at 04:11 UTC e23a43aef8bc51c5201658775445f529324ed728
1 file changed +5
vllm/model_executor/models/gpt_neox.py
+5
@@ -274,6 +274,11 @@ class GPTNeoXForCausalLM(nn.Module):
274 if ("attention.bias" in name or "attention.masked_bias" in name
275 or "rotary_emb.inv_freq" in name):
276 continue
277 + if ("rotary_emb.cos_cached" in name
278 + or "rotary_emb.sin_cached" in name):
279 + # Models trained using OpenRLHF may include
280 + # these tensors in the checkpoint. Skip them.
281 + continue
282 param = params_dict[name]
283
284 if "query_key_value" in name: