@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 91a61da9b1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Bugfix] fix load local safetensors model (#2512)

Roy committed Jan 20, 2024 at 08:26 UTC 91a61da9b12c483a6688841b8f860c1a32b8918c
1 file changed +2 -2
vllm/model_executor/weight_utils.py
+2 -2
@@ -159,8 +159,6 @@ def prepare_hf_model_weights(
159 matching = fnmatch.filter(file_list, pattern)
160 if len(matching) > 0:
161 allow_patterns = [pattern]
162 - if pattern == "*.safetensors":
163 - use_safetensors = True
162 break
163
164 logger.info(f"Using model weights format {allow_patterns}")
@@ -178,6 +176,8 @@ def prepare_hf_model_weights(
176 for pattern in allow_patterns:
177 hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
178 if len(hf_weights_files) > 0:
179 + if pattern == "*.safetensors":
180 + use_safetensors = True
181 break
182 if not use_safetensors:
183 # Exclude files that are not needed for inference.