A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Use hidden_size_per_head as head_size fallback (#24221)
Signed-off-by: nopperl <54780682+nopperl@users.noreply.github.com>
nopperl committed
Sep 4, 2025 at 20:59 UTC
2b30afa4420cbada6dd9084de3ee7eb19142b7ff
1 file changed
+5
vllm/config/__init__.py
+5
@@ -1426,6 +1426,11 @@ class ModelConfig:
1426
if getattr(self.hf_text_config, "head_dim", None) is not None:
1427
return self.hf_text_config.head_dim
1428
1429
+ # NOTE: Some models (such as PLaMo2.1) use `hidden_size_per_head`
1430
+ if getattr(self.hf_text_config, "hidden_size_per_head",
1431
+ None) is not None:
1432
+ return self.hf_text_config.hidden_size_per_head
1433
+
1434
# FIXME(woosuk): This may not be true for all models.
1435
return (self.hf_text_config.hidden_size //
1436
self.hf_text_config.num_attention_heads)