A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Fix KV head calculation for MPT models when using GQA (#5142)
Bruce Fontaine committed
Jun 17, 2024 at 15:26 UTC
a3e8a05d4c1b79dd44eb92bb6f57eb40c3fbdb21
1 file changed
+5
-1
vllm/config.py
+5
-1
@@ -302,7 +302,11 @@ class ModelConfig:
302
return 1
303
304
# For DBRX and MPT
305
- if self.hf_config.model_type in ["dbrx", "mpt"]:
305
+ if self.hf_config.model_type == "mpt":
306
+ if "kv_n_heads" in self.hf_config.attn_config:
307
+ return self.hf_config.attn_config["kv_n_heads"]
308
+ return self.hf_config.num_attention_heads
309
+ if self.hf_config.model_type == "dbrx":
310
return getattr(self.hf_config.attn_config, "kv_n_heads",
311
self.hf_config.num_attention_heads)
312