@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e519ae097a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

add tqdm when loading checkpoint shards (#6569)

Co-authored-by: tianyi.zhao <tianyi.zhao@transwarp.io> Co-authored-by: youkaichao <youkaichao@126.com>

zhaotyer committed Jul 23, 2024 at 11:48 UTC e519ae097ac4c126d3d8b2f3bb5ee54f696cf8bc
1 file changed +6 -3
vllm/model_executor/model_loader/weight_utils.py
+6 -3
@@ -331,7 +331,8 @@ def np_cache_weights_iterator(
331 with get_lock(model_name_or_path, cache_dir):
332 if not os.path.exists(weight_names_file):
333 weight_names: List[str] = []
334 - for bin_file in hf_weights_files:
334 + for bin_file in tqdm(hf_weights_files,
335 + desc="Loading np_cache checkpoint shards"):
336 state = torch.load(bin_file, map_location="cpu")
337 for name, param in state.items():
338 param_path = os.path.join(np_folder, name)
@@ -355,7 +356,8 @@ def safetensors_weights_iterator(
356 hf_weights_files: List[str]
357 ) -> Generator[Tuple[str, torch.Tensor], None, None]:
358 """Iterate over the weights in the model safetensor files."""
358 - for st_file in hf_weights_files:
359 + for st_file in tqdm(hf_weights_files,
360 + desc="Loading safetensors checkpoint shards"):
361 with safe_open(st_file, framework="pt") as f:
362 for name in f.keys(): # noqa: SIM118
363 param = f.get_tensor(name)
@@ -366,7 +368,8 @@ def pt_weights_iterator(
368 hf_weights_files: List[str]
369 ) -> Generator[Tuple[str, torch.Tensor], None, None]:
370 """Iterate over the weights in the model bin/pt files."""
369 - for bin_file in hf_weights_files:
371 + for bin_file in tqdm(hf_weights_files,
372 + desc="Loading pt checkpoint shards"):
373 state = torch.load(bin_file, map_location="cpu")
374 for name, param in state.items():
375 yield name, param