@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c4774eb841
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix] Fix snapshot download in serving benchmark (#6318)

Roger Wang committed Jul 11, 2024 at 00:04 UTC c4774eb8418864390341d35103aa747fc411b59c
1 file changed +9 -9
benchmarks/backend_request_func.py
+9 -9
@@ -390,17 +390,17 @@ def remove_prefix(text: str, prefix: str) -> str:
390 return text
391
392
393 -def get_model(pretrained_model_name_or_path: str):
393 +def get_model(pretrained_model_name_or_path: str) -> str:
394 if os.getenv('VLLM_USE_MODELSCOPE', 'False').lower() == 'true':
395 from modelscope import snapshot_download
396 - else:
397 - from huggingface_hub import snapshot_download
398 -
399 - model_path = snapshot_download(
400 - model_id=pretrained_model_name_or_path,
401 - local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
402 - ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"])
403 - return model_path
396 +
397 + model_path = snapshot_download(
398 + model_id=pretrained_model_name_or_path,
399 + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
400 + ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"])
401 +
402 + return model_path
403 + return pretrained_model_name_or_path
404
405
406 def get_tokenizer(