A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 07:00 UTC
[Misc] Turn off encoder torch compile by default (#28634)
Signed-off-by: Roger Wang <hey@rogerw.io>
Roger Wang committed
Nov 13, 2025 at 08:38 UTC
d3387750f191f3bcf6607db95436147bbccfacb3
3 files changed
+11
-5
tests/compile/test_multimodal_compile.py
+6
-3
@@ -10,8 +10,8 @@ from vllm.platforms import current_platform
10
11
def test_compile():
12
vllm_config = VllmConfig()
13
- # Default configuration compiles mm encoder
14
- assert vllm_config.compilation_config.compile_mm_encoder
13
+ # Default configuration does not compile mm encoder
14
+ assert not vllm_config.compilation_config.compile_mm_encoder
15
16
17
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
@@ -39,7 +39,10 @@ def test_qwen2_5_vl_compilation(vllm_runner, monkeypatch):
39
"Qwen/Qwen2.5-VL-3B-Instruct",
40
max_model_len=2048,
41
gpu_memory_utilization=0.8,
42
- compilation_config={"mode": CompilationMode.VLLM_COMPILE},
42
+ compilation_config={
43
+ "mode": CompilationMode.VLLM_COMPILE,
44
+ "compile_mm_encoder": True,
45
+ },
46
) as _,
47
):
48
pass
tests/models/multimodal/generation/test_common.py
+2
@@ -131,6 +131,7 @@ VLM_TEST_SETTINGS = {
131
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
132
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
133
video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",
134
+ enforce_eager=False,
135
max_model_len=4096,
136
max_num_seqs=2,
137
auto_cls=AutoModelForImageTextToText,
@@ -160,6 +161,7 @@ VLM_TEST_SETTINGS = {
161
VLMTestType.MULTI_IMAGE,
162
VLMTestType.VIDEO,
163
),
164
+ enforce_eager=False,
165
needs_video_metadata=True,
166
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
167
img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
vllm/config/compilation.py
+3
-2
@@ -320,9 +320,10 @@ class CompilationConfig:
320
321
If None, defaults to attention ops for piecewise cudagraphs.
322
If empty list [], no ops are excluded (suitable for full cudagraphs)."""
323
- compile_mm_encoder: bool = True
323
+ compile_mm_encoder: bool = False
324
"""Whether or not to compile the multimodal encoder.
325
- Currently, this only works for `Qwen2_5_vl`."""
325
+ Currently, this only works for `Qwen2_5_vl` on selected platforms.
326
+ Disabled by default until more models are supported/tested to work."""
327
328
# Inductor capture
329
use_inductor: bool | None = None