A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Bugfix] fix load local safetensors model (#2512)
Roy committed
Jan 20, 2024 at 08:26 UTC
91a61da9b12c483a6688841b8f860c1a32b8918c
1 file changed
+2
-2
vllm/model_executor/weight_utils.py
+2
-2
@@ -159,8 +159,6 @@ def prepare_hf_model_weights(
159
matching = fnmatch.filter(file_list, pattern)
160
if len(matching) > 0:
161
allow_patterns = [pattern]
162
- if pattern == "*.safetensors":
163
- use_safetensors = True
162
break
163
164
logger.info(f"Using model weights format {allow_patterns}")
@@ -178,6 +176,8 @@ def prepare_hf_model_weights(
176
for pattern in allow_patterns:
177
hf_weights_files += glob.glob(os.path.join(hf_folder, pattern))
178
if len(hf_weights_files) > 0:
179
+ if pattern == "*.safetensors":
180
+ use_safetensors = True
181
break
182
if not use_safetensors:
183
# Exclude files that are not needed for inference.