A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
fix stablelm.py tensor-parallel-size bug (#2482)
YingchaoX committed
Jan 19, 2024 at 01:39 UTC
8a25d3a71ac8b0b06c37935ed67e7c35aa901bf5
1 file changed
+1
-1
vllm/model_executor/models/stablelm.py
+1
-1
@@ -99,7 +99,7 @@ class StablelmAttention(nn.Module):
99
self.q_size = self.num_heads * self.head_dim
100
self.kv_size = self.num_key_value_heads * self.head_dim
101
102
- if (self.head_dim * self.num_heads) != self.hidden_size:
102
+ if (self.head_dim * self.num_heads * tp_size) != self.hidden_size:
103
raise ValueError(
104
f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
105
f" and `num_heads`: {self.num_heads}).")