A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Use head_dim in config if exists (#2622)
Xiang Xu committed
Jan 27, 2024 at 10:30 UTC
220a47627bf48c728ce0a2737be39c400bb6f653
1 file changed
+2
vllm/config.py
+2
@@ -213,6 +213,8 @@ class ModelConfig:
213
return self.hf_config.hidden_size
214
215
def get_head_size(self) -> int:
216
+ if hasattr(self.hf_config, "head_dim"):
217
+ return self.hf_config.head_dim
218
# FIXME(woosuk): This may not be true for all models.
219
return self.hf_config.hidden_size // self.hf_config.num_attention_heads
220