| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from functools import partial |
| 4 | |
| 5 | import pytest |
| 6 | |
| 7 | from tests.models.utils import dummy_hf_overrides |
| 8 | from vllm.compilation.counter import compilation_counter |
| 9 | from vllm.config import VllmConfig |
| 10 | from vllm.config.compilation import CompilationMode, CUDAGraphMode |
| 11 | from vllm.platforms import current_platform |
| 12 | |
| 13 | |
| 14 | def test_compile(): |
| 15 | vllm_config = VllmConfig() |
| 16 | # Default configuration does not compile mm encoder |
| 17 | assert not vllm_config.compilation_config.compile_mm_encoder |
| 18 | |
| 19 | |
| 20 | @pytest.mark.forked |
| 21 | @pytest.mark.core_model |
| 22 | @pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda") |
| 23 | @pytest.mark.parametrize( |
| 24 | ("model", "tokenizer", "config_format", "tokenizer_mode", "model_arch"), |
| 25 | [ |
| 26 | pytest.param( |
| 27 | "mistralai/Pixtral-12B-2409", |
| 28 | None, |
| 29 | "mistral", |
| 30 | "mistral", |
| 31 | "PixtralForConditionalGeneration", |
| 32 | id="pixtral", |
| 33 | ), |
| 34 | pytest.param( |
| 35 | "mistralai/Mistral-Small-3.1-24B-Instruct-2503", |
| 36 | "mistral-community/pixtral-12b", |
| 37 | "hf", |
| 38 | "auto", |
| 39 | "Mistral3ForConditionalGeneration", |
| 40 | id="mistral3-hf", |
| 41 | ), |
| 42 | ], |
| 43 | ) |
| 44 | @pytest.mark.parametrize( |
| 45 | ("compile_mm_encoder", "expected_models"), |
| 46 | [ |
| 47 | # The dummy model has two compiled text components. Enabling encoder |
| 48 | # compilation adds its single vision transformer block. |
| 49 | pytest.param(False, 2, id="disabled"), |
| 50 | pytest.param(True, 3, id="enabled"), |
| 51 | ], |
| 52 | ) |
| 53 | def test_pixtral_compilation( |
| 54 | vllm_runner, |
| 55 | monkeypatch, |
| 56 | model: str, |
| 57 | tokenizer: str | None, |
| 58 | config_format: str, |
| 59 | tokenizer_mode: str, |
| 60 | model_arch: str, |
| 61 | compile_mm_encoder: bool, |
| 62 | expected_models: int, |
| 63 | ): |
| 64 | """Test compilation for both Pixtral vision encoder implementations.""" |
| 65 | monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") |
| 66 | monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0") |
| 67 | |
| 68 | with ( |
| 69 | compilation_counter.expect(num_models_seen=expected_models), |
| 70 | vllm_runner( |
| 71 | model, |
| 72 | tokenizer_name=tokenizer, |
| 73 | tokenizer_mode=tokenizer_mode, |
| 74 | config_format=config_format, |
| 75 | load_format="dummy", |
| 76 | hf_overrides=partial(dummy_hf_overrides, model_arch=model_arch), |
| 77 | max_model_len=4096, |
| 78 | limit_mm_per_prompt={"image": 1}, |
| 79 | gpu_memory_utilization=0.8, |
| 80 | attention_backend="FLASH_ATTN", |
| 81 | compilation_config={ |
| 82 | "mode": CompilationMode.VLLM_COMPILE, |
| 83 | "compile_mm_encoder": compile_mm_encoder, |
| 84 | "cudagraph_mode": CUDAGraphMode.NONE, |
| 85 | }, |
| 86 | ) as _, |
| 87 | ): |
| 88 | pass |
| 89 | |
| 90 | |
| 91 | # forked needed to workaround https://github.com/vllm-project/vllm/issues/21073 |
| 92 | @pytest.mark.forked |
| 93 | @pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda") |
| 94 | def test_qwen2_5_vl_compilation(vllm_runner, monkeypatch): |
| 95 | """Test that Qwen2.5-VL vision submodules are compiled. |
| 96 | |
| 97 | This test verifies that the 3 vision submodules (Qwen2_5_VisionPatchEmbed, |
| 98 | Qwen2_5_VisionBlock, and Qwen2_5_VisionPatchMerger) are properly tagged |
| 99 | for compilation by checking that num_models_seen increases by at least 3. |
| 100 | """ |
| 101 | # Disable multiprocessing so that the counter is in the same process |
| 102 | monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") |
| 103 | |
| 104 | with ( |
| 105 | # NOTE: Qwen2.5-VL has 35 models in total - the LLM backend |
| 106 | # Vision Patch Embed, Vision Patch Merger, and then 32 Vision Blocks |
| 107 | # (one for each layer) - in the future, we should fix vLLM compilation |
| 108 | # logic to handle this case and only compile the Vision submodules once |
| 109 | # and reuse the compiled code for all layers |
| 110 | # See https://github.com/vllm-project/vllm/issues/27590 |
| 111 | compilation_counter.expect(num_models_seen=35), |
| 112 | vllm_runner( |
| 113 | "Qwen/Qwen2.5-VL-3B-Instruct", |
| 114 | max_model_len=2048, |
| 115 | gpu_memory_utilization=0.8, |
| 116 | compilation_config={ |
| 117 | "mode": CompilationMode.VLLM_COMPILE, |
| 118 | "compile_mm_encoder": True, |
| 119 | }, |
| 120 | ) as _, |
| 121 | ): |
| 122 | pass |
| 123 | |
| 124 | |
| 125 | # forked needed to workaround https://github.com/vllm-project/vllm/issues/21073 |
| 126 | @pytest.mark.forked |
| 127 | @pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda") |
| 128 | def test_qwen2_5_vl_no_vit_compilation(vllm_runner, monkeypatch): |
| 129 | """Test that Qwen2.5-VL vision submodules are not compiled when the |
| 130 | config is passed off |
| 131 | """ |
| 132 | # Disable multiprocessing so that the counter is in the same process |
| 133 | monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") |
| 134 | |
| 135 | with ( |
| 136 | compilation_counter.expect(num_models_seen=1), |
| 137 | vllm_runner( |
| 138 | "Qwen/Qwen2.5-VL-3B-Instruct", |
| 139 | max_model_len=2048, |
| 140 | gpu_memory_utilization=0.8, |
| 141 | compilation_config={ |
| 142 | "mode": CompilationMode.VLLM_COMPILE, |
| 143 | "compile_mm_encoder": False, |
| 144 | }, |
| 145 | ) as _, |
| 146 | ): |
| 147 | pass |
| 148 | |
| 149 | |
| 150 | # forked needed to workaround https://github.com/vllm-project/vllm/issues/21073 |
| 151 | # Requires Cuda and 8 gpus as well |
| 152 | @pytest.mark.forked |
| 153 | @pytest.mark.skip(reason="Skipping due to CI resource constraints") |
| 154 | def test_mllama4_vit_compilation(vllm_runner, monkeypatch): |
| 155 | """Test that Mllama4 vision submodules are compiled. |
| 156 | |
| 157 | This test verifies that the 2 vision submodules (Llama4VisionEncoder, |
| 158 | Llama4VisionPixelShuffleMLP) are properly tagged |
| 159 | for compilation by checking that num_models_seen increases to 3. |
| 160 | |
| 161 | However since we are using TP=8, we compilation_counter will not |
| 162 | work properly so we will just check the run succeeds rn |
| 163 | """ |
| 164 | # Disable multiprocessing so that the counter is in the same process |
| 165 | monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") |
| 166 | |
| 167 | with ( |
| 168 | monkeypatch.context(), |
| 169 | # TODO: Since we require TP=8, this messes with the compilation |
| 170 | # counter. We should fix this in the future, but leave for now |
| 171 | # to make sure that compilation runs (no crash) with llama vision encoder |
| 172 | compilation_counter.expect(num_models_seen=0), |
| 173 | vllm_runner( |
| 174 | "meta-llama/Llama-4-Scout-17B-16E-Instruct", |
| 175 | max_model_len=512, |
| 176 | gpu_memory_utilization=0.8, |
| 177 | tensor_parallel_size=8, |
| 178 | compilation_config={ |
| 179 | "mode": CompilationMode.VLLM_COMPILE, |
| 180 | "compile_mm_encoder": True, |
| 181 | }, |
| 182 | ), |
| 183 | ): |
| 184 | pass |