A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 11:15 UTC
[Model Runner v2] Fix v2 compile counter `num_gpu_runner_capture_triggers` and `num_cudagraph_captured` (#41285)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Apr 30, 2026 at 18:20 UTC
526927be94c63ee677840c9074754fb6fdbdf5a1
2 files changed
+5
vllm/v1/worker/gpu/cudagraph_utils.py
+2
@@ -9,6 +9,7 @@ import torch
9
import torch.nn as nn
10
from tqdm import tqdm
11
12
+from vllm.compilation.counter import compilation_counter
13
from vllm.config import VllmConfig
14
from vllm.config.compilation import CUDAGraphMode
15
from vllm.distributed.parallel_state import (
@@ -237,6 +238,7 @@ class CudaGraphManager:
238
# the next forward pass.
239
get_offloader().join_after_forward()
240
self.graphs[desc] = graph
241
+ compilation_counter.num_cudagraph_captured += 1
242
self._graphs_captured = True
243
return captured_attn_states
244
vllm/v1/worker/gpu/model_runner.py
+3
@@ -27,6 +27,7 @@ import numpy as np
27
import torch
28
import torch.nn as nn
29
30
+from vllm.compilation.counter import compilation_counter
31
from vllm.config import VllmConfig
32
from vllm.config.compilation import CUDAGraphMode
33
from vllm.distributed.parallel_state import (
@@ -587,6 +588,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
588
)
589
return 0
590
591
+ compilation_counter.num_gpu_runner_capture_triggers += 1
592
+
593
start_time = time.perf_counter()
594
gc.collect()
595
torch.accelerator.empty_cache()