A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core] Sharded State Loader download from HF (#4889)
Aurick Qiao committed
May 20, 2024 at 14:46 UTC
1937e29848c8de8634c5421612d57863aa0e2a51
1 file changed
+15
-1
vllm/model_executor/model_loader/loader.py
+15
-1
@@ -423,6 +423,16 @@ class ShardedStateLoader(BaseModelLoader):
423
result[k] = t
424
return result
425
426
+ def _prepare_weights(self, model_name_or_path: str,
427
+ revision: Optional[str]):
428
+ if os.path.isdir(model_name_or_path):
429
+ return model_name_or_path
430
+ else:
431
+ allow_patterns = ["*.safetensors"]
432
+ return download_weights_from_hf(model_name_or_path,
433
+ self.load_config.download_dir,
434
+ allow_patterns, revision)
435
+
436
def load_model(self, *, model_config: ModelConfig,
437
device_config: DeviceConfig,
438
lora_config: Optional[LoRAConfig],
@@ -433,6 +443,10 @@ class ShardedStateLoader(BaseModelLoader):
443
from safetensors.torch import safe_open
444
445
from vllm.distributed import get_tensor_model_parallel_rank
446
+
447
+ local_model_path = self._prepare_weights(model_config.model,
448
+ model_config.revision)
449
+
450
with set_default_torch_dtype(model_config.dtype):
451
with torch.device(device_config.device):
452
model = _initialize_model(model_config, self.load_config,
@@ -440,7 +454,7 @@ class ShardedStateLoader(BaseModelLoader):
454
cache_config)
455
rank = get_tensor_model_parallel_rank()
456
pattern = os.path.join(
443
- model_config.model,
457
+ local_model_path,
458
self.pattern.format(rank=rank, part="*"),
459
)
460
filepaths = glob.glob(pattern)