A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Change the load format to pt for Mixtral (#2028)
Woosuk Kwon committed
Dec 11, 2023 at 10:32 UTC
b9bcdc715808c2ec110a6e98e98e4fbe0681f8bf
1 file changed
+10
vllm/config.py
+10
@@ -119,6 +119,16 @@ class ModelConfig:
119
# Force ROCm to load from pt weights if nothing specific is set
120
if load_format == "auto":
121
load_format = "pt"
122
+
123
+ # FIXME(woosuk): This is a temporary hack. Support safetensor weights.
124
+ architectures = getattr(self.hf_config, "architectures", [])
125
+ if "MixtralForCausalLM" in architectures and load_format != "pt":
126
+ logger.info(
127
+ "Currently, only 'pt' format is supported for Mixtral. "
128
+ "Changing the format to 'pt'. This may re-download the "
129
+ "weights if you have downloaded the safetensor weights.")
130
+ load_format = "pt"
131
+
132
self.load_format = load_format
133
134
def _verify_tokenizer_mode(self) -> None: