[ROCm][CI] Enable ViT CUDA graph tests on AMD gfx950 GPUs (#52256)
Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Shanshan Shen committed
Aug 17, 2026 at 09:52 UTC
0ad04cff1b10267a2642becbf31f62c91018cad5
4 files changed
+71
-30
.buildkite/test-amd.yaml
+5
-2
@@ -2092,7 +2092,7 @@ steps:
2092
- tests/models/multimodal/generation
2093
- tests/models/multimodal/test_mapping.py
2094
commands:
2095
- - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
2095
+ - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
2096
- pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
2097
- cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
2098
@@ -3671,7 +3671,8 @@ steps:
3671
- vllm/
3672
- tests/models/multimodal/generation
3673
commands:
3674
- - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
3674
+ - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
3675
+ - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
3676
- pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
3677
- cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
3678
@@ -3771,6 +3772,7 @@ steps:
3772
- tests/v1/executor
3773
- tests/v1/kv_offload
3774
- tests/v1/worker
3775
+ - tests/v1/cudagraph
3776
- tests/v1/kv_connector/unit
3777
- tests/v1/metrics
3778
- tests/entrypoints/openai/correctness/test_lmeval.py
@@ -3780,6 +3782,7 @@ steps:
3782
- pytest -v -s v1/executor
3783
- pytest -v -s v1/kv_offload
3784
- pytest -v -s v1/worker
3785
+ - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py
3786
- pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
3787
- pytest -v -s -m 'not cpu_test' v1/metrics
3788
- pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api
docs/design/cuda_graphs_multimodal.md
+54
-24
@@ -7,6 +7,59 @@ For two-tower vision encoders (e.g., DeepSeek-OCR's SAM + CLIP with dynamic tili
7
!!! note
8
Encoder CUDA Graphs are orthogonal to decoder CUDA Graphs — both can be enabled simultaneously. Encoder graphs capture the vision encoder execution (e.g., ViT in Qwen3-VL), while decoder graphs capture the language model execution as described in the [CUDA Graphs design document](cuda_graphs.md).
9
10
+## Compatibility Matrix
11
+
12
+!!! note
13
+ The symbols used below have the following meanings:
14
+
15
+ - ✅ = Full compatibility
16
+ - 🟠 = Partial compatibility
17
+ - ❌ = No compatibility
18
+ - ❔ = Unknown or TBD
19
+
20
+### Model x Feature
21
+
22
+| Architecture | Models | CG for Image | CG for Video | Multi-Path Graph |
23
+| ------------ | ------ | ------------ | ------------ | --------------- |
24
+| `DeepseekOCRForCausalLM` | `DeepSeek-OCR` | ✅︎ | ❌︎ | ✅︎ |
25
+| `Ernie4_5_VLMoeForConditionalGeneration` | `ERNIE-4.5-VL` | ✅︎ | ❌︎ | ❌︎ |
26
+| `Gemma3ForConditionalGeneration` | `Gemma3` | ✅︎ | ❌︎ | ❌︎ |
27
+| `Glm4vForConditionalGeneration` | `GLM-4.1V, GLM-4.6V-Flash` | ✅︎ | ✅︎ | ❌︎ |
28
+| `Gemma4ForConditionalGeneration` | `Gemma-4` | ✅︎ | ✅︎ | ❌︎ |
29
+| `InternVLChatModel` | `InternVL3.5`, `InternVL3`, `InternVL2.5`, `InternVL2` | ✅︎ | ✅︎ | ❌︎ |
30
+| `KimiVLForConditionalGeneration` | `Kimi-VL` | ✅︎ | ❌︎ | ❌︎ |
31
+| `Llama4ForConditionalGeneration` | `Llama 4` | ✅︎ | ❌︎ | ❌︎ |
32
+| `Qwen2VLForConditionalGeneration` | `Qwen2-VL` | ✅︎ | ✅︎ | ❌︎ |
33
+| `Qwen2_5_VLForConditionalGeneration` | `Qwen2.5-VL` | ✅︎ | ✅︎ | ❌︎ |
34
+| `Qwen3VLForConditionalGeneration` | `Qwen3-VL` | ✅︎ | ✅︎ | ❌︎ |
35
+| `Qwen3_5ForConditionalGeneration` | `Qwen3.5`, `Qwen3.6` | ✅︎ | ✅︎ | ❌︎ |
36
+| `Qwen3_5MoeForConditionalGeneration` | `Qwen3.5-MoE`, `Qwen3.6-MoE` | ✅︎ | ✅︎ | ❌︎ |
37
+| `Step3VLForConditionalGeneration` | `Step3-VL` | ✅︎ | ❌︎ | ✅︎ |
38
+
39
+### Model x Hardware
40
+
41
+| Architecture | NV Blackwell | NV Ampere | AMD MI300X | AMD MI350X / MI355X |
42
+| ------------ | ---------------- | ------------- | -------------- | --------------------- |
43
+| `DeepseekOCRForCausalLM` | ✅︎ | ✅︎ | ❔ | ✅︎ |
44
+| `Ernie4_5_VLMoeForConditionalGeneration` | ✅︎ | ✅︎ | ❔ | ✅︎ |
45
+| `Gemma3ForConditionalGeneration` | ✅︎ | ✅︎ | ❔ | ✅︎ |
46
+| `Glm4vForConditionalGeneration` | ✅︎ | ✅︎ | ❔ | ✅︎ |
47
+| `Gemma4ForConditionalGeneration` | ✅︎ | ✅︎ | ❔ | ✅︎ |
48
+| `InternVLChatModel` | ✅︎ | ✅︎ | ❔ | ✅︎ |
49
+| `KimiVLForConditionalGeneration` | ✅︎ | ✅︎ | ❔ | ✅︎ |
50
+| `Llama4ForConditionalGeneration` | ✅︎ | ✅︎ | ❔ | ✅︎ |
51
+| `Qwen2VLForConditionalGeneration` | ✅︎ | ✅︎ | ❔ | ✅︎ |
52
+| `Qwen2_5_VLForConditionalGeneration` | ✅︎ | ✅︎ | ❔ | ✅︎ |
53
+| `Qwen3VLForConditionalGeneration` | ✅︎ | ✅︎ | ❔ | ✅︎ |
54
+| `Qwen3_5ForConditionalGeneration` | ✅︎ | ✅︎ | ❔ | ✅︎ |
55
+| `Qwen3_5MoeForConditionalGeneration` | ✅︎ | ✅︎ | ❔ | ✅︎ |
56
+| `Step3VLForConditionalGeneration` | ✅︎ | ✅︎ | ❔ | ✅︎ |
57
+
58
+!!! note
59
+ Encoder CUDA Graph has currently been tested with `--mm-encoder-attn-backend=FLASH_ATTN` and `--mm-encoder-attn-backend=FLASHINFER` on Blackwell GPUs.
60
+ For Qwen2-VL and Qwen2.5-VL only FA2 and FA3 has been tested.
61
+ Encoder CUDA Graph has also been tested with AMD MI350X (gfx950) used `--mm-encoder-attn-backend=FLASH_ATTN` (the ROCm default).
62
+
63
## Motivation
64
65
Vision encoder inference incurs CUDA kernel launch overhead on the host side. The overhead is more significant when the batch size is small or image size is small.
@@ -113,29 +166,6 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra
166
!!! note
167
The `SupportsEncoderCudaGraph` protocol is designed to be model-agnostic. New vision encoder models can opt-in by implementing the protocol methods without modifying the manager.
168
116
-**Supported models:**
117
-
118
-| Architecture | Models | CG for Image | CG for Video | Dual-Path Graph |
119
-| ------------ | ------ | ------------ | ------------ | --------------- |
120
-| `DeepseekOCRForCausalLM` | `DeepSeek-OCR` | ✅︎ | ❌︎ | ✅︎ |
121
-| `Ernie4_5_VLMoeForConditionalGeneration` | `ERNIE-4.5-VL` | ✅︎ | ❌︎ | ❌︎ |
122
-| `Gemma3ForConditionalGeneration` | `Gemma3` | ✅︎ | ❌︎ | ❌︎ |
123
-| `Glm4vForConditionalGeneration` | `GLM-4.1V, GLM-4.6V-Flash` | ✅︎ | ✅︎ | ❌︎ |
124
-| `Gemma4ForConditionalGeneration` | `Gemma-4` | ✅︎ | ✅︎ | ❌︎ |
125
-| `InternVLChatModel` | `InternVL3.5`, `InternVL3`, `InternVL2.5`, `InternVL2` | ✅︎ | ✅︎ | ❌︎ |
126
-| `KimiVLForConditionalGeneration` | `Kimi-VL` | ✅︎ | ❌︎ | ❌︎ |
127
-| `Llama4ForConditionalGeneration` | `Llama 4` | ✅︎ | ❌︎ | ❌︎ |
128
-| `Qwen2VLForConditionalGeneration` | `Qwen2-VL` | ✅︎ | ✅︎ | ❌︎ |
129
-| `Qwen2_5_VLForConditionalGeneration` | `Qwen2.5-VL` | ✅︎ | ✅︎ | ❌︎ |
130
-| `Qwen3VLForConditionalGeneration` | `Qwen3-VL` | ✅︎ | ✅︎ | ❌︎ |
131
-| `Qwen3_5ForConditionalGeneration` | `Qwen3.5`, `Qwen3.6` | ✅︎ | ✅︎ | ❌︎ |
132
-| `Qwen3_5MoeForConditionalGeneration` | `Qwen3.5-MoE`, `Qwen3.6-MoE` | ✅︎ | ✅︎ | ❌︎ |
133
-| `Step3VLForConditionalGeneration` | `Step3-VL` | ✅︎ | ❌︎ | ✅︎ |
134
-
135
-!!! note
136
- Encoder CUDA Graphs have currently been tested with `--mm-encoder-attn-backend=FLASH_ATTN` and `--mm-encoder-attn-backend=FLASHINFER` on Blackwell GPUs.
137
- For Qwen2-VL and Qwen2.5-VL only FA2 and FA3 has been tested.
138
-
169
## Configuration
170
171
Four fields in `CompilationConfig` control encoder CUDA Graphs:
@@ -228,7 +258,7 @@ model = vllm.LLM(
258
)
259
```
260
231
-## About the Performance
261
+## Benchmark Results
262
263
The following benchmarks were run on Blackwell GPUs (GB200) using `vllm bench mm-processor`. See [#35963](https://github.com/vllm-project/vllm/pull/35963) for full details.
264
tests/models/multimodal/generation/test_vit_cudagraph.py
+6
-2
@@ -313,7 +313,9 @@ def get_compilation_config(config: VitCudagraphTestConfig):
313
314
315
@pytest.mark.parametrize("model_id", params_with_marks(MODEL_CONFIGS))
316
-@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
316
+@pytest.mark.skipif(
317
+ not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
318
+)
319
def test_vit_cudagraph_image(model_id, vllm_runner, image_assets):
320
config = MODEL_CONFIGS[model_id]
321
@@ -357,7 +359,9 @@ def test_vit_cudagraph_image(model_id, vllm_runner, image_assets):
359
360
361
@pytest.mark.parametrize("model_id", params_with_marks(MODEL_CONFIGS))
360
-@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
362
+@pytest.mark.skipif(
363
+ not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
364
+)
365
def test_vit_cudagraph_video(model_id, vllm_runner, video_assets):
366
config = MODEL_CONFIGS[model_id]
367
tests/v1/cudagraph/test_encoder_cudagraph.py
+6
-2
@@ -475,7 +475,9 @@ def _make_video_mm_kwargs(
475
# ---------------------------------------------------------------------------
476
477
478
-@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
478
+@pytest.mark.skipif(
479
+ not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
480
+)
481
class TestEncoderCudaGraphCaptureReplay:
482
def setup_method(self):
483
self.device = torch.device("cuda:0")
@@ -758,7 +760,9 @@ _VIDEO_MAX_BATCH = 4
760
_VIDEO_MAX_FRAMES = 8 # 2 frames per item at max_batch_size=4
761
762
761
-@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
763
+@pytest.mark.skipif(
764
+ not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
765
+)
766
class TestEncoderCudaGraphVideoReplay:
767
def setup_method(self):
768
self.device = torch.device("cuda:0")