@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b880ffb87e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Add tqdm progress bar during graph capture (#11349)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Dec 19, 2024 at 23:35 UTC b880ffb87e0bcde5e3693203b480df49e46d67bc
1 file changed +13 -5
vllm/worker/model_runner.py
+13 -5
@@ -13,6 +13,7 @@ import numpy as np
13 import torch
14 import torch.distributed
15 import torch.nn as nn
16 +from tqdm import tqdm
17
18 import vllm.envs as envs
19 from vllm.attention import AttentionMetadata, get_attn_backend
@@ -21,7 +22,8 @@ from vllm.attention.backends.utils import CommonAttentionState
22 from vllm.config import CompilationLevel, VllmConfig
23 from vllm.core.scheduler import SchedulerOutputs
24 from vllm.distributed import get_kv_transfer_group, get_pp_group
24 -from vllm.distributed.parallel_state import graph_capture
25 +from vllm.distributed.parallel_state import (get_tensor_model_parallel_rank,
26 + graph_capture)
27 from vllm.forward_context import set_forward_context
28 from vllm.inputs import INPUT_REGISTRY, InputRegistry
29 from vllm.logger import init_logger
@@ -1413,8 +1415,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1415 logger.info("Capturing cudagraphs for decoding. This may lead to "
1416 "unexpected consequences if the model is not static. To "
1417 "run the model in eager mode, set 'enforce_eager=True' or "
1416 - "use '--enforce-eager' in the CLI.")
1417 - logger.info("If out-of-memory error occurs during cudagraph capture,"
1418 + "use '--enforce-eager' in the CLI. "
1419 + "If out-of-memory error occurs during cudagraph capture,"
1420 " consider decreasing `gpu_memory_utilization` or "
1421 "switching to eager mode. You can also reduce the "
1422 "`max_num_seqs` as needed to decrease memory usage.")
@@ -1451,8 +1453,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1453 # memory usage of CUDA graph.
1454 for virtual_engine in range(
1455 self.parallel_config.pipeline_parallel_size):
1454 - for batch_size in \
1455 - self.vllm_config.compilation_config.capture_sizes:
1456 + # Only rank 0 should print progress bar during capture
1457 + capture_sizes = (
1458 + tqdm(
1459 + self.vllm_config.compilation_config.capture_sizes,
1460 + desc="Capturing CUDA graph shapes",
1461 + ) if get_tensor_model_parallel_rank() == 0 else
1462 + self.vllm_config.compilation_config.capture_sizes)
1463 + for batch_size in capture_sizes:
1464 attn_metadata = (
1465 self.attn_state.graph_capture_get_metadata_for_batch(
1466 batch_size,