A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
fix: incorrect bigcode attention heads num (#676)
Wen Sun committed
Aug 5, 2023 at 01:35 UTC
621980bdc0d5a41e224febf962a6e0474e2b14ef
1 file changed
+5
-3
vllm/config.py
+5
-3
@@ -98,9 +98,11 @@ class ModelConfig:
98
# Note: for falcon, when new_decoder_architecture is True, the
99
# multi_query flag is ignored and we use n_head_kv for the number of
100
# KV heads.
101
- if (getattr(self.hf_config, "multi_query", False) and
102
- (self.hf_config.model_type == "falcon" and
103
- not getattr(self.hf_config, "new_decoder_architecture", False))):
101
+ new_decoder_arch_falcon = (
102
+ self.hf_config.model_type == "falcon"
103
+ and getattr(self.hf_config, "new_decoder_architecture", False))
104
+ if not new_decoder_arch_falcon and getattr(self.hf_config,
105
+ "multi_query", False):
106
# Multi-query attention, only one KV head.
107
return 1
108
# For Falcon: