A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] handle hf_config with architectures == None (#3982)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Co-authored-by: Simon Mo <simon.mo@hey.com>
Travis Johnson committed
Apr 10, 2024 at 16:28 UTC
934d3662f716d60abfb04cf9fdd6d20f6e75f140
1 file changed
+3
-1
vllm/config.py
+3
-1
@@ -158,7 +158,9 @@ class ModelConfig:
158
159
# TODO: Remove this check once HF updates the pt weights of Mixtral.
160
architectures = getattr(self.hf_config, "architectures", [])
161
- if "MixtralForCausalLM" in architectures and load_format == "pt":
161
+ # architectures can be None instead of []
162
+ if architectures and "MixtralForCausalLM" in architectures \
163
+ and load_format == "pt":
164
raise ValueError(
165
"Currently, the 'pt' format is not supported for Mixtral. "
166
"Please use the 'safetensors' format instead. ")