A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] Fix snapshot download in serving benchmark (#6318)
Roger Wang committed
Jul 11, 2024 at 00:04 UTC
c4774eb8418864390341d35103aa747fc411b59c
1 file changed
+9
-9
benchmarks/backend_request_func.py
+9
-9
@@ -390,17 +390,17 @@ def remove_prefix(text: str, prefix: str) -> str:
390
return text
391
392
393
-def get_model(pretrained_model_name_or_path: str):
393
+def get_model(pretrained_model_name_or_path: str) -> str:
394
if os.getenv('VLLM_USE_MODELSCOPE', 'False').lower() == 'true':
395
from modelscope import snapshot_download
396
- else:
397
- from huggingface_hub import snapshot_download
398
-
399
- model_path = snapshot_download(
400
- model_id=pretrained_model_name_or_path,
401
- local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
402
- ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"])
403
- return model_path
396
+
397
+ model_path = snapshot_download(
398
+ model_id=pretrained_model_name_or_path,
399
+ local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
400
+ ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"])
401
+
402
+ return model_path
403
+ return pretrained_model_name_or_path
404
405
406
def get_tokenizer(