@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 621980bdc0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

fix: incorrect bigcode attention heads num (#676)

Wen Sun committed Aug 5, 2023 at 01:35 UTC 621980bdc0d5a41e224febf962a6e0474e2b14ef
1 file changed +5 -3
vllm/config.py
+5 -3
@@ -98,9 +98,11 @@ class ModelConfig:
98 # Note: for falcon, when new_decoder_architecture is True, the
99 # multi_query flag is ignored and we use n_head_kv for the number of
100 # KV heads.
101 - if (getattr(self.hf_config, "multi_query", False) and
102 - (self.hf_config.model_type == "falcon" and
103 - not getattr(self.hf_config, "new_decoder_architecture", False))):
101 + new_decoder_arch_falcon = (
102 + self.hf_config.model_type == "falcon"
103 + and getattr(self.hf_config, "new_decoder_architecture", False))
104 + if not new_decoder_arch_falcon and getattr(self.hf_config,
105 + "multi_query", False):
106 # Multi-query attention, only one KV head.
107 return 1
108 # For Falcon: