A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[misc] only tqdm for first rank (#6672)
youkaichao committed
Jul 22, 2024 at 21:57 UTC
c5201240a4ffb40d014845088eb218b1512899e9
1 file changed
+31
-6
vllm/model_executor/model_loader/weight_utils.py
+31
-6
@@ -313,6 +313,13 @@ def filter_files_not_needed_for_inference(
313
return hf_weights_files
314
315
316
+# explicitly use pure text format, with a newline at the end
317
+# this makes it impossible to see the animation in the progress bar
318
+# but will avoid messing up with ray or multiprocessing, which wraps
319
+# each line of output with some prefix.
320
+_BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501
321
+
322
+
323
def np_cache_weights_iterator(
324
model_name_or_path: str, cache_dir: Optional[str], hf_folder: str,
325
hf_weights_files: List[str]
@@ -321,6 +328,8 @@ def np_cache_weights_iterator(
328
329
Will dump the model weights to numpy files if they are not already dumped.
330
"""
331
+ enable_tqdm = not torch.distributed.is_initialized(
332
+ ) or torch.distributed.get_rank() == 0
333
# Convert the model weights from torch tensors to numpy arrays for
334
# faster loading.
335
np_folder = os.path.join(hf_folder, "np")
@@ -331,8 +340,12 @@ def np_cache_weights_iterator(
340
with get_lock(model_name_or_path, cache_dir):
341
if not os.path.exists(weight_names_file):
342
weight_names: List[str] = []
334
- for bin_file in tqdm(hf_weights_files,
335
- desc="Loading np_cache checkpoint shards"):
343
+ for bin_file in tqdm(
344
+ hf_weights_files,
345
+ desc="Loading np_cache checkpoint shards",
346
+ disable=not enable_tqdm,
347
+ bar_format=_BAR_FORMAT,
348
+ ):
349
state = torch.load(bin_file, map_location="cpu")
350
for name, param in state.items():
351
param_path = os.path.join(np_folder, name)
@@ -356,8 +369,14 @@ def safetensors_weights_iterator(
369
hf_weights_files: List[str]
370
) -> Generator[Tuple[str, torch.Tensor], None, None]:
371
"""Iterate over the weights in the model safetensor files."""
359
- for st_file in tqdm(hf_weights_files,
360
- desc="Loading safetensors checkpoint shards"):
372
+ enable_tqdm = not torch.distributed.is_initialized(
373
+ ) or torch.distributed.get_rank() == 0
374
+ for st_file in tqdm(
375
+ hf_weights_files,
376
+ desc="Loading safetensors checkpoint shards",
377
+ disable=not enable_tqdm,
378
+ bar_format=_BAR_FORMAT,
379
+ ):
380
with safe_open(st_file, framework="pt") as f:
381
for name in f.keys(): # noqa: SIM118
382
param = f.get_tensor(name)
@@ -368,8 +387,14 @@ def pt_weights_iterator(
387
hf_weights_files: List[str]
388
) -> Generator[Tuple[str, torch.Tensor], None, None]:
389
"""Iterate over the weights in the model bin/pt files."""
371
- for bin_file in tqdm(hf_weights_files,
372
- desc="Loading pt checkpoint shards"):
390
+ enable_tqdm = not torch.distributed.is_initialized(
391
+ ) or torch.distributed.get_rank() == 0
392
+ for bin_file in tqdm(
393
+ hf_weights_files,
394
+ desc="Loading pt checkpoint shards",
395
+ disable=not enable_tqdm,
396
+ bar_format=_BAR_FORMAT,
397
+ ):
398
state = torch.load(bin_file, map_location="cpu")
399
for name, param in state.items():
400
yield name, param