A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
fix: CUDA error when inferencing with Falcon-40B base model (#992)
Kyujin Cho committed
Sep 10, 2023 at 17:39 UTC
898285c9bf306e32c7f161b75bf0bf7fd483f265
1 file changed
+2
-1
vllm/config.py
+2
-1
@@ -114,8 +114,9 @@ class ModelConfig:
114
# Note: for falcon, when new_decoder_architecture is True, the
115
# multi_query flag is ignored and we use n_head_kv for the number of
116
# KV heads.
117
+ falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"]
118
new_decoder_arch_falcon = (
118
- self.hf_config.model_type == "falcon"
119
+ self.hf_config.model_type in falcon_model_types
120
and getattr(self.hf_config, "new_decoder_architecture", False))
121
if not new_decoder_arch_falcon and getattr(self.hf_config,
122
"multi_query", False):