A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
GPTJConfig has no attribute rotary. (#532)
leegohi04517 committed
Jul 25, 2023 at 02:29 UTC
1dde34e0f8f1021103e62fdf81ae7e89cce5aaf2
1 file changed
+1
-1
vllm/model_executor/models/gpt_j.py
+1
-1
@@ -65,7 +65,7 @@ class GPTJAttention(nn.Module):
65
self.num_heads = self.total_num_heads // tp_world_size
66
67
scaling = self.head_size**-0.5
68
- assert config.rotary
68
+ assert getattr(config, "rotary", True)
69
assert config.rotary_dim % 2 == 0
70
self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_size,
71
scaling, config.rotary_dim)