@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1dde34e0f8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

GPTJConfig has no attribute rotary. (#532)

leegohi04517 committed Jul 25, 2023 at 02:29 UTC 1dde34e0f8f1021103e62fdf81ae7e89cce5aaf2
1 file changed +1 -1
vllm/model_executor/models/gpt_j.py
+1 -1
@@ -65,7 +65,7 @@ class GPTJAttention(nn.Module):
65 self.num_heads = self.total_num_heads // tp_world_size
66
67 scaling = self.head_size**-0.5
68 - assert config.rotary
68 + assert getattr(config, "rotary", True)
69 assert config.rotary_dim % 2 == 0
70 self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_size,
71 scaling, config.rotary_dim)