@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a7b3e33078
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[Fix] Fix RoPE in ChatGLM-32K (#1841)

Woosuk Kwon committed Nov 29, 2023 at 13:01 UTC a7b3e33078469943d2a11b1c3d634e220b71bf76
1 file changed +5
vllm/model_executor/models/chatglm.py
+5
@@ -75,12 +75,17 @@ class GLMAttention(nn.Module):
75 linear_method=linear_method,
76 )
77
78 + # https://huggingface.co/THUDM/chatglm3-6b-32k/blob/e210410255278dd9d74463cf396ba559c0ef801c/modeling_chatglm.py#L141
79 + rope_ratio = getattr(config, "rope_ratio", 1.0)
80 + max_positions = getattr(config, "seq_length", 8192)
81 self.attn = PagedAttentionWithRoPE(
82 self.num_heads,
83 self.head_dim,
84 self.scaling,
85 rotary_dim=self.head_dim // 2,
86 num_kv_heads=self.num_kv_heads,
87 + max_position=max_positions,
88 + base=10000 * rope_ratio,
89 is_neox_style=False,
90 )
91