A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 04:15 UTC
[CI] Fix "test_vit_cudagraph_[image|video][step3_vl]" failure (#43082)
Signed-off-by: haosdent <haosdent@gmail.com>
haosdent committed
May 21, 2026 at 12:22 UTC
7e5070934ee5f28103c5b95cb776904a12fc36f5
1 file changed
+26
-4
tests/models/multimodal/generation/test_vit_cudagraph.py
+26
-4
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from dataclasses import dataclass, field
5
+from functools import partial
6
7
import pytest
8
@@ -10,6 +11,7 @@ from vllm.platforms import current_platform
11
12
from ....conftest import IMAGE_ASSETS, VIDEO_ASSETS
13
from ....utils import create_new_process_for_each_test
14
+from ...utils import dummy_hf_overrides
15
from .vlm_utils.builders import sample_frames_with_video_metadata
16
17
@@ -26,6 +28,7 @@ class VitCudagraphTestConfig:
28
num_video_frames: int = 16
29
needs_video_metadata: bool = False
30
vllm_runner_kwargs: dict = field(default_factory=dict)
31
+ compilation_config_overrides: dict = field(default_factory=dict)
32
marks: list = field(default_factory=list)
33
34
@@ -99,17 +102,36 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
102
),
103
"step3_vl": VitCudagraphTestConfig(
104
model="stepfun-ai/Step3-VL-10B",
105
+ modalities=["image"],
106
image_prompt=step3_vl_chat_template("What is in this image?"),
103
- video_prompt=None,
107
+ # Single bucket sized to cover the largest test image's output
108
+ # tokens (1152 > 1141 for cherry_blossom). The default auto-
109
+ # inferred range fans out into multiple power-of-2 buckets, each
110
+ # holding a full ViT capture pool.
111
+ compilation_config_overrides={
112
+ "encoder_cudagraph_token_budgets": [1152],
113
+ },
114
+ # Shrink to 1 text + 1 vision layer with random weights so the
115
+ # test runs on any CI GPU (incl. L4) and skips the 20 GiB weight
116
+ # download. The test only validates that encoder CG capture/
117
+ # replay functions correctly, not output quality.
118
+ vllm_runner_kwargs={
119
+ "load_format": "dummy",
120
+ "hf_overrides": partial(
121
+ dummy_hf_overrides,
122
+ model_arch="StepVLForConditionalGeneration",
123
+ ),
124
+ },
125
),
126
}
127
128
108
-def get_compilation_config():
129
+def get_compilation_config(config: VitCudagraphTestConfig):
130
return {
131
"cudagraph_mm_encoder": True,
132
"encoder_cudagraph_max_vision_items_per_batch": 1,
133
"encoder_cudagraph_max_frames_per_batch": 16,
134
+ **config.compilation_config_overrides,
135
}
136
137
@@ -141,7 +163,7 @@ def test_vit_cudagraph_image(model_id, vllm_runner, image_assets):
163
max_model_len=config.max_model_len,
164
max_num_seqs=config.max_num_seqs,
165
limit_mm_per_prompt={"image": 1},
144
- compilation_config=get_compilation_config(),
166
+ compilation_config=get_compilation_config(config),
167
**config.vllm_runner_kwargs,
168
) as vllm_model:
169
outputs = vllm_model.generate_greedy(
@@ -194,7 +216,7 @@ def test_vit_cudagraph_video(model_id, vllm_runner, video_assets):
216
max_model_len=config.max_model_len,
217
max_num_seqs=config.max_num_seqs,
218
limit_mm_per_prompt={"video": 1},
197
- compilation_config=get_compilation_config(),
219
+ compilation_config=get_compilation_config(config),
220
**config.vllm_runner_kwargs,
221
) as vllm_model:
222
outputs = vllm_model.generate_greedy(