A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[Fix] Fix RoPE in ChatGLM-32K (#1841)
Woosuk Kwon committed
Nov 29, 2023 at 13:01 UTC
a7b3e33078469943d2a11b1c3d634e220b71bf76
1 file changed
+5
vllm/model_executor/models/chatglm.py
+5
@@ -75,12 +75,17 @@ class GLMAttention(nn.Module):
75
linear_method=linear_method,
76
)
77
78
+ # https://huggingface.co/THUDM/chatglm3-6b-32k/blob/e210410255278dd9d74463cf396ba559c0ef801c/modeling_chatglm.py#L141
79
+ rope_ratio = getattr(config, "rope_ratio", 1.0)
80
+ max_positions = getattr(config, "seq_length", 8192)
81
self.attn = PagedAttentionWithRoPE(
82
self.num_heads,
83
self.head_dim,
84
self.scaling,
85
rotary_dim=self.head_dim // 2,
86
num_kv_heads=self.num_kv_heads,
87
+ max_position=max_positions,
88
+ base=10000 * rope_ratio,
89
is_neox_style=False,
90
)
91