A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Fix KeyError on loading GPT-NeoX (#3925)
Junichi Sato committed
Apr 10, 2024 at 04:11 UTC
e23a43aef8bc51c5201658775445f529324ed728
1 file changed
+5
vllm/model_executor/models/gpt_neox.py
+5
@@ -274,6 +274,11 @@ class GPTNeoXForCausalLM(nn.Module):
274
if ("attention.bias" in name or "attention.masked_bias" in name
275
or "rotary_emb.inv_freq" in name):
276
continue
277
+ if ("rotary_emb.cos_cached" in name
278
+ or "rotary_emb.sin_cached" in name):
279
+ # Models trained using OpenRLHF may include
280
+ # these tensors in the checkpoint. Skip them.
281
+ continue
282
param = params_dict[name]
283
284
if "query_key_value" in name: