@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d036198e23
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[BugFix][Model] Fix commandr RoPE max_position_embeddings (#3919)

Roy committed Apr 9, 2024 at 06:17 UTC d036198e23345f3c25438f082396f7487028e8b6
1 file changed +3 -1
vllm/model_executor/models/commandr.py
+3 -1
@@ -140,7 +140,9 @@ class CohereAttention(nn.Module):
140 self.q_size = self.num_heads * self.head_dim
141 self.kv_size = self.num_kv_heads * self.head_dim
142 self.scaling = self.head_dim**-0.5
143 - self.max_position_embeddings = config.max_position_embeddings
143 + self.max_position_embeddings = getattr(
144 + config, "model_max_length", None) or getattr(
145 + config, "max_position_embeddings", 8192)
146 self.rope_theta = config.rope_theta
147 self.rope_scaling = getattr(config, "rope_scaling", None)
148 self.use_qk_norm = getattr(config, "use_qk_norm", False)