@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b9bcdc7158
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Change the load format to pt for Mixtral (#2028)

Woosuk Kwon committed Dec 11, 2023 at 10:32 UTC b9bcdc715808c2ec110a6e98e98e4fbe0681f8bf
1 file changed +10
vllm/config.py
+10
@@ -119,6 +119,16 @@ class ModelConfig:
119 # Force ROCm to load from pt weights if nothing specific is set
120 if load_format == "auto":
121 load_format = "pt"
122 +
123 + # FIXME(woosuk): This is a temporary hack. Support safetensor weights.
124 + architectures = getattr(self.hf_config, "architectures", [])
125 + if "MixtralForCausalLM" in architectures and load_format != "pt":
126 + logger.info(
127 + "Currently, only 'pt' format is supported for Mixtral. "
128 + "Changing the format to 'pt'. This may re-download the "
129 + "weights if you have downloaded the safetensor weights.")
130 + load_format = "pt"
131 +
132 self.load_format = load_format
133
134 def _verify_tokenizer_mode(self) -> None: