A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 19:00 UTC
[MM][Perf][CG] Support ViT full CUDA graph for Qwen3.5 (#42151)
Signed-off-by: shen-shanshan <467638484@qq.com>
Shanshan Shen committed
May 13, 2026 at 16:00 UTC
92def124bcb74424384d7941bff2fb5b72d6d615
4 files changed
+112
-5
docs/design/cuda_graphs_multimodal.md
+2
-1
@@ -85,8 +85,9 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra
85
86
| Architecture | Models | CG for Image | CG for Video |
87
| ------------ | ------ | ------------ | ------------ |
88
-| `Qwen3VLForConditionalGeneration` | `Qwen3-VL` | ✅︎ | ✅︎ |
88
| `Qwen2_5_VLForConditionalGeneration` | `Qwen2.5-VL` | ✅︎ | ✅︎ |
89
+| `Qwen3VLForConditionalGeneration` | `Qwen3-VL` | ✅︎ | ✅︎ |
90
+| `Qwen3_5ForConditionalGeneration` | `Qwen3.5` | ✅︎ | ✅︎ |
91
92
!!! note
93
Encoder CUDA Graphs have currently been tested with `--mm-encoder-attn-backend=FLASH_ATTN` and `--mm-encoder-attn-backend=FLASHINFER` on Blackwell GPUs.
examples/generate/multimodal/vision_language_offline.py
+93
-1
@@ -2179,6 +2179,92 @@ def run_qwen3_vl_moe(questions: list[str], modality: str) -> ModelRequestData:
2179
)
2180
2181
2182
+# Qwen3.5-Dense
2183
+def run_qwen3_5(questions: list[str], modality: str) -> ModelRequestData:
2184
+ model_name = "Qwen/Qwen3.5-4B"
2185
+
2186
+ mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
2187
+ engine_args = EngineArgs(
2188
+ model=model_name,
2189
+ max_model_len=4096,
2190
+ max_num_seqs=5,
2191
+ mm_processor_kwargs={
2192
+ "min_pixels": 28 * 28,
2193
+ "max_pixels": 1280 * 28 * 28,
2194
+ "fps": 1,
2195
+ },
2196
+ limit_mm_per_prompt=mm_limit,
2197
+ )
2198
+
2199
+ image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
2200
+ video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"
2201
+
2202
+ if modality == "image":
2203
+ placeholder = image_placeholder
2204
+ elif modality == "video":
2205
+ placeholder = video_placeholder
2206
+ elif modality == "image+video":
2207
+ placeholder = image_placeholder + video_placeholder
2208
+
2209
+ prompts = [
2210
+ (
2211
+ "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
2212
+ f"<|im_start|>user\n{placeholder}"
2213
+ f"{question}<|im_end|>\n"
2214
+ "<|im_start|>assistant\n"
2215
+ )
2216
+ for question in questions
2217
+ ]
2218
+
2219
+ return ModelRequestData(
2220
+ engine_args=engine_args,
2221
+ prompts=prompts,
2222
+ )
2223
+
2224
+
2225
+# Qwen3.5-MoE
2226
+def run_qwen3_5_moe(questions: list[str], modality: str) -> ModelRequestData:
2227
+ model_name = "Qwen/Qwen3.5-35B-A3B"
2228
+
2229
+ mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
2230
+ engine_args = EngineArgs(
2231
+ model=model_name,
2232
+ max_model_len=4096,
2233
+ max_num_seqs=5,
2234
+ mm_processor_kwargs={
2235
+ "min_pixels": 28 * 28,
2236
+ "max_pixels": 1280 * 28 * 28,
2237
+ "fps": 1,
2238
+ },
2239
+ limit_mm_per_prompt=mm_limit,
2240
+ )
2241
+
2242
+ image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
2243
+ video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"
2244
+
2245
+ if modality == "image":
2246
+ placeholder = image_placeholder
2247
+ elif modality == "video":
2248
+ placeholder = video_placeholder
2249
+ elif modality == "image+video":
2250
+ placeholder = image_placeholder + video_placeholder
2251
+
2252
+ prompts = [
2253
+ (
2254
+ "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
2255
+ f"<|im_start|>user\n{placeholder}"
2256
+ f"{question}<|im_end|>\n"
2257
+ "<|im_start|>assistant\n"
2258
+ )
2259
+ for question in questions
2260
+ ]
2261
+
2262
+ return ModelRequestData(
2263
+ engine_args=engine_args,
2264
+ prompts=prompts,
2265
+ )
2266
+
2267
+
2268
# R-4B
2269
def run_r_vl(questions: list[str], modality: str) -> ModelRequestData:
2270
assert modality == "image"
@@ -2442,6 +2528,8 @@ model_example_map = {
2528
"qwen2_5_omni": run_qwen2_5_omni,
2529
"qwen3_vl": run_qwen3_vl,
2530
"qwen3_vl_moe": run_qwen3_vl_moe,
2531
+ "qwen3_5": run_qwen3_5,
2532
+ "qwen3_5_moe": run_qwen3_5_moe,
2533
"rvl": run_r_vl,
2534
"skywork_chat": run_skyworkr1v,
2535
"smolvlm": run_smolvlm,
@@ -2460,13 +2548,17 @@ MODELS_NEED_VIDEO_METADATA = [
2548
"molmo2",
2549
"qwen3_vl",
2550
"qwen3_vl_moe",
2551
+ "qwen3_5",
2552
+ "qwen3_5_moe",
2553
]
2554
2555
2556
MODELS_SUPPORT_VIT_CUDA_GRAPH = [
2557
+ "qwen2_5_vl",
2558
"qwen3_vl",
2559
"qwen3_vl_moe",
2469
- "qwen2_5_vl",
2560
+ "qwen3_5",
2561
+ "qwen3_5_moe",
2562
]
2563
2564
tests/models/multimodal/generation/test_vit_cudagraph.py
+15
-3
@@ -42,6 +42,18 @@ def qwen_vl_chat_template(content: str) -> str:
42
43
44
MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
45
+ "qwen2_5_vl": VitCudagraphTestConfig(
46
+ model="Qwen/Qwen2.5-VL-3B-Instruct",
47
+ image_prompt=qwen_vl_chat_template(
48
+ "<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
49
+ ),
50
+ video_prompt=qwen_vl_chat_template(
51
+ "<|vision_start|><|video_pad|><|vision_end|>"
52
+ "Describe this video in one sentence."
53
+ ),
54
+ needs_video_metadata=False,
55
+ marks=[pytest.mark.core_model],
56
+ ),
57
"qwen3_vl": VitCudagraphTestConfig(
58
model="Qwen/Qwen3-VL-2B-Instruct",
59
image_prompt=qwen_vl_chat_template(
@@ -54,8 +66,8 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
66
needs_video_metadata=True,
67
marks=[pytest.mark.core_model],
68
),
57
- "qwen2_5_vl": VitCudagraphTestConfig(
58
- model="Qwen/Qwen2.5-VL-3B-Instruct",
69
+ "qwen3_5": VitCudagraphTestConfig(
70
+ model="Qwen/Qwen3.5-0.8B",
71
image_prompt=qwen_vl_chat_template(
72
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
73
),
@@ -63,7 +75,7 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
75
"<|vision_start|><|video_pad|><|vision_end|>"
76
"Describe this video in one sentence."
77
),
66
- needs_video_metadata=False,
78
+ needs_video_metadata=True,
79
marks=[pytest.mark.core_model],
80
),
81
}
vllm/model_executor/models/qwen3_5.py
+2
@@ -565,6 +565,7 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
565
multimodal_config = vllm_config.model_config.multimodal_config
566
567
self.config = config
568
+ self.model_config = vllm_config.model_config
569
self.multimodal_config = multimodal_config
570
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
571
# Qwen3.5 does not support multimodal pruning (EVS).
@@ -778,6 +779,7 @@ class Qwen3_5MoeForConditionalGeneration(
779
multimodal_config = vllm_config.model_config.multimodal_config
780
781
self.config = config
782
+ self.model_config = vllm_config.model_config
783
self.multimodal_config = multimodal_config
784
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
785
# Qwen3.5 does not support multimodal pruning (EVS).