A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Add tqdm progress bar during graph capture (#11349)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Dec 19, 2024 at 23:35 UTC
b880ffb87e0bcde5e3693203b480df49e46d67bc
1 file changed
+13
-5
vllm/worker/model_runner.py
+13
-5
@@ -13,6 +13,7 @@ import numpy as np
13
import torch
14
import torch.distributed
15
import torch.nn as nn
16
+from tqdm import tqdm
17
18
import vllm.envs as envs
19
from vllm.attention import AttentionMetadata, get_attn_backend
@@ -21,7 +22,8 @@ from vllm.attention.backends.utils import CommonAttentionState
22
from vllm.config import CompilationLevel, VllmConfig
23
from vllm.core.scheduler import SchedulerOutputs
24
from vllm.distributed import get_kv_transfer_group, get_pp_group
24
-from vllm.distributed.parallel_state import graph_capture
25
+from vllm.distributed.parallel_state import (get_tensor_model_parallel_rank,
26
+ graph_capture)
27
from vllm.forward_context import set_forward_context
28
from vllm.inputs import INPUT_REGISTRY, InputRegistry
29
from vllm.logger import init_logger
@@ -1413,8 +1415,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1415
logger.info("Capturing cudagraphs for decoding. This may lead to "
1416
"unexpected consequences if the model is not static. To "
1417
"run the model in eager mode, set 'enforce_eager=True' or "
1416
- "use '--enforce-eager' in the CLI.")
1417
- logger.info("If out-of-memory error occurs during cudagraph capture,"
1418
+ "use '--enforce-eager' in the CLI. "
1419
+ "If out-of-memory error occurs during cudagraph capture,"
1420
" consider decreasing `gpu_memory_utilization` or "
1421
"switching to eager mode. You can also reduce the "
1422
"`max_num_seqs` as needed to decrease memory usage.")
@@ -1451,8 +1453,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1453
# memory usage of CUDA graph.
1454
for virtual_engine in range(
1455
self.parallel_config.pipeline_parallel_size):
1454
- for batch_size in \
1455
- self.vllm_config.compilation_config.capture_sizes:
1456
+ # Only rank 0 should print progress bar during capture
1457
+ capture_sizes = (
1458
+ tqdm(
1459
+ self.vllm_config.compilation_config.capture_sizes,
1460
+ desc="Capturing CUDA graph shapes",
1461
+ ) if get_tensor_model_parallel_rank() == 0 else
1462
+ self.vllm_config.compilation_config.capture_sizes)
1463
+ for batch_size in capture_sizes:
1464
attn_metadata = (
1465
self.attn_state.graph_capture_get_metadata_for_batch(
1466
batch_size,