A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
add tqdm when loading checkpoint shards (#6569)
Co-authored-by: tianyi.zhao <tianyi.zhao@transwarp.io> Co-authored-by: youkaichao <youkaichao@126.com>
zhaotyer committed
Jul 23, 2024 at 11:48 UTC
e519ae097ac4c126d3d8b2f3bb5ee54f696cf8bc
1 file changed
+6
-3
vllm/model_executor/model_loader/weight_utils.py
+6
-3
@@ -331,7 +331,8 @@ def np_cache_weights_iterator(
331
with get_lock(model_name_or_path, cache_dir):
332
if not os.path.exists(weight_names_file):
333
weight_names: List[str] = []
334
- for bin_file in hf_weights_files:
334
+ for bin_file in tqdm(hf_weights_files,
335
+ desc="Loading np_cache checkpoint shards"):
336
state = torch.load(bin_file, map_location="cpu")
337
for name, param in state.items():
338
param_path = os.path.join(np_folder, name)
@@ -355,7 +356,8 @@ def safetensors_weights_iterator(
356
hf_weights_files: List[str]
357
) -> Generator[Tuple[str, torch.Tensor], None, None]:
358
"""Iterate over the weights in the model safetensor files."""
358
- for st_file in hf_weights_files:
359
+ for st_file in tqdm(hf_weights_files,
360
+ desc="Loading safetensors checkpoint shards"):
361
with safe_open(st_file, framework="pt") as f:
362
for name in f.keys(): # noqa: SIM118
363
param = f.get_tensor(name)
@@ -366,7 +368,8 @@ def pt_weights_iterator(
368
hf_weights_files: List[str]
369
) -> Generator[Tuple[str, torch.Tensor], None, None]:
370
"""Iterate over the weights in the model bin/pt files."""
369
- for bin_file in hf_weights_files:
371
+ for bin_file in tqdm(hf_weights_files,
372
+ desc="Loading pt checkpoint shards"):
373
state = torch.load(bin_file, map_location="cpu")
374
for name, param in state.items():
375
yield name, param