@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8a25d3a71a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

fix stablelm.py tensor-parallel-size bug (#2482)

YingchaoX committed Jan 19, 2024 at 01:39 UTC 8a25d3a71ac8b0b06c37935ed67e7c35aa901bf5
1 file changed +1 -1
vllm/model_executor/models/stablelm.py
+1 -1
@@ -99,7 +99,7 @@ class StablelmAttention(nn.Module):
99 self.q_size = self.num_heads * self.head_dim
100 self.kv_size = self.num_key_value_heads * self.head_dim
101
102 - if (self.head_dim * self.num_heads) != self.hidden_size:
102 + if (self.head_dim * self.num_heads * tp_size) != self.hidden_size:
103 raise ValueError(
104 f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
105 f" and `num_heads`: {self.num_heads}).")