A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 21:15 UTC
[MM][CG] Optimize default `max_frames_per_batch` auto-infer for ViT CUDA graph video inference (#40445)
Signed-off-by: shen-shanshan <467638484@qq.com>
Shanshan Shen committed
Apr 21, 2026 at 22:47 UTC
936e0b79aa93008bb90e4cc190ea8adb9296b1df
6 files changed
+46
-15
docs/design/cuda_graphs_multimodal.md
+2
-1
@@ -76,6 +76,7 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra
76
* `encoder_cudagraph_forward(...)` — forward pass using precomputed buffers (called during capture and replay).
77
* `encoder_eager_forward(...)` — fallback eager forward when no graph fits.
78
* `get_input_modality(...)` - return the modality of the inputs.
79
+* `get_max_frames_per_video()` - return model-specific max frames per video.
80
81
!!! note
82
The `SupportsEncoderCudaGraph` protocol is designed to be model-agnostic. New vision encoder models can opt-in by implementing the protocol methods without modifying the manager.
@@ -96,7 +97,7 @@ Three fields in `CompilationConfig` control encoder CUDA Graphs:
97
* `cudagraph_mm_encoder` (`bool`, default `False`) — enable CUDA Graph capture for multimodal encoder. When enabled, captures the full encoder forward as a CUDA Graph for each token budget level.
98
* `encoder_cudagraph_token_budgets` (`list[int]`, default `[]`) — token budget levels for capture. If empty (default), auto-inferred from model architecture as power-of-2 levels. User-provided values override auto-inference.
99
* `encoder_cudagraph_max_vision_items_per_batch` (`int`, default `0`) — maximum number of images/videos per batch during capture. If 0 (default), auto-inferred as `max_budget // min_budget`.
99
-* `encoder_cudagraph_max_frames_per_batch` (`int`, default `0`) — maximum number of video frames per batch during capture. If 0 (default), auto-inferred as `encoder_cudagraph_max_vision_items_per_batch * 2` (to be optimized).
100
+* `encoder_cudagraph_max_frames_per_batch` (`int`, default `None`) — maximum number of video frames per batch during capture. If `None` (default), auto-inferred as `encoder_cudagraph_max_vision_items_per_batch * max_frames_per_video` (`max_frames_per_video` is a model-specific value according to its `processing_info`). If we limit the video count per prompt to `0`, it will also be set to `0` (i.e., fall back to image-only mode).
101
102
## Usage guide
103
vllm/config/compilation.py
+9
-6
@@ -535,13 +535,15 @@ class CompilationConfig:
535
model's budget range. User-provided positive value overrides
536
auto-inference."""
537
538
- encoder_cudagraph_max_frames_per_batch: int = 0
538
+ encoder_cudagraph_max_frames_per_batch: int | None = None
539
"""Maximum total video frames per batch for encoder CUDA graph capture.
540
Controls the cu_seqlens buffer size (one entry per attention sequence,
541
- i.e. one per video frame). If 0 (default), auto-inferred per budget
542
- level as token_budget (tight bound: packing guarantees
543
- sum(T_i) <= token_budget). Positive value overrides auto-inference
544
- and applies to all budget levels."""
541
+ i.e. one per video frame).
542
+ If None (default), auto-inferred as encoder_cudagraph_max_vision_items_per_batch
543
+ * max_frames_per_video (model-specific value according to processing_info).
544
+ Positive value overrides auto-inference and applies to all budget levels.
545
+ If we limit the video count per prompt to `0`, it will also be set to `0`
546
+ (i.e., fall back to image-only mode)."""
547
548
# Inductor capture
549
compile_sizes: list[int | str] | None = None
@@ -993,11 +995,12 @@ class CompilationConfig:
995
)
996
if (
997
self.cudagraph_mm_encoder
998
+ and self.encoder_cudagraph_max_frames_per_batch is not None
999
and self.encoder_cudagraph_max_frames_per_batch < 0
1000
):
1001
raise ValueError(
1002
"encoder_cudagraph_max_frames_per_batch must be "
1000
- "non-negative (0 = auto-infer)"
1003
+ "non-negative (None = auto-infer)"
1004
)
1005
1006
if self.backend == "":
vllm/model_executor/models/interfaces.py
+6
@@ -1531,6 +1531,12 @@ class SupportsEncoderCudaGraph(Protocol):
1531
"""Return the modality of the inputs."""
1532
...
1533
1534
+ def get_max_frames_per_video(
1535
+ self,
1536
+ ) -> int:
1537
+ """Return model-specific max frames per video."""
1538
+ ...
1539
+
1540
def get_encoder_cudagraph_budget_range(
1541
self,
1542
vllm_config: "VllmConfig",
vllm/model_executor/models/qwen3_vl.py
+10
@@ -1640,6 +1640,7 @@ class Qwen3VLForConditionalGeneration(
1640
multimodal_config = vllm_config.model_config.multimodal_config
1641
1642
self.config = config
1643
+ self.model_config = vllm_config.model_config
1644
self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
1645
self.multimodal_config = multimodal_config
1646
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
@@ -1783,6 +1784,15 @@ class Qwen3VLForConditionalGeneration(
1784
return "image"
1785
return "video"
1786
1787
+ def get_max_frames_per_video(self) -> int:
1788
+ mm_registry = MULTIMODAL_REGISTRY
1789
+ info = mm_registry.get_processing_info(self.model_config)
1790
+ max_frames_per_video = info.get_num_frames_with_most_features(
1791
+ seq_len=self.model_config.max_model_len,
1792
+ mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
1793
+ )
1794
+ return max_frames_per_video
1795
+
1796
def get_encoder_cudagraph_budget_range(
1797
self,
1798
vllm_config,
vllm/multimodal/registry.py
+3
@@ -193,6 +193,9 @@ class MultiModalRegistry:
193
ctx = self._create_processing_ctx(model_config, tokenizer)
194
return factories.info(ctx)
195
196
+ def get_processing_info(self, model_config: "ModelConfig") -> BaseProcessingInfo:
197
+ return self._create_processing_info(model_config, tokenizer=None)
198
+
199
def create_processor(
200
self,
201
model_config: "ModelConfig",
vllm/v1/worker/encoder_cudagraph.py
+16
-8
@@ -67,13 +67,15 @@ class EncoderCudaGraphManager:
67
68
comp_config = vllm_config.compilation_config
69
user_budgets = comp_config.encoder_cudagraph_token_budgets
70
- user_max_mm_items = comp_config.encoder_cudagraph_max_vision_items_per_batch
70
+ user_max_vision_items = comp_config.encoder_cudagraph_max_vision_items_per_batch
71
user_max_frames = comp_config.encoder_cudagraph_max_frames_per_batch
72
73
- if user_budgets and user_max_mm_items > 0:
73
+ multimodal_config = vllm_config.model_config.multimodal_config
74
+
75
+ if user_budgets and user_max_vision_items > 0:
76
# Fully user-specified
77
self.token_budgets = sorted(user_budgets)
76
- self.max_batch_size = user_max_mm_items
78
+ self.max_batch_size = user_max_vision_items
79
else:
80
# Auto-infer missing values from model
81
min_budget, max_budget = model.get_encoder_cudagraph_budget_range(
@@ -85,14 +87,20 @@ class EncoderCudaGraphManager:
87
else self._generate_budgets(min_budget, max_budget)
88
)
89
self.max_batch_size = (
88
- user_max_mm_items if user_max_mm_items > 0 else max_budget // min_budget
90
+ user_max_vision_items
91
+ if user_max_vision_items > 0
92
+ else max_budget // min_budget
93
)
94
91
- if user_max_frames > 0:
95
+ assert multimodal_config is not None
96
+ if multimodal_config.get_limit_per_prompt("video") == 0:
97
+ self.max_frames_per_batch = 0
98
+ elif user_max_frames is not None:
99
self.max_frames_per_batch = user_max_frames
100
else:
94
- # TODO(shen-shanshan): optimize this auto-infer for max_frames_per_batch.
95
- self.max_frames_per_batch = self.max_batch_size * 2
101
+ # Set it to the model-specific value according to its `processing_info`.
102
+ max_frames_per_video = self.model.get_max_frames_per_video()
103
+ self.max_frames_per_batch = self.max_batch_size * max_frames_per_video
104
105
mm_config = vllm_config.model_config.multimodal_config
106
self.use_dp = (
@@ -111,7 +119,7 @@ class EncoderCudaGraphManager:
119
"budgets=%s, max_batch_size=%d, max_frames_per_batch=%s, use_dp=%s",
120
self.token_budgets,
121
self.max_batch_size,
114
- self.max_frames_per_batch if self.max_frames_per_batch > 0 else "auto",
122
+ self.max_frames_per_batch,
123
self.use_dp,
124
)
125