@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 898285c9bf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

fix: CUDA error when inferencing with Falcon-40B base model (#992)

Kyujin Cho committed Sep 10, 2023 at 17:39 UTC 898285c9bf306e32c7f161b75bf0bf7fd483f265
1 file changed +2 -1
vllm/config.py
+2 -1
@@ -114,8 +114,9 @@ class ModelConfig:
114 # Note: for falcon, when new_decoder_architecture is True, the
115 # multi_query flag is ignored and we use n_head_kv for the number of
116 # KV heads.
117 + falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"]
118 new_decoder_arch_falcon = (
118 - self.hf_config.model_type == "falcon"
119 + self.hf_config.model_type in falcon_model_types
120 and getattr(self.hf_config, "new_decoder_architecture", False))
121 if not new_decoder_arch_falcon and getattr(self.hf_config,
122 "multi_query", False):