A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 15:45 UTC
Fix issues from #28242 (#28257)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Nov 6, 2025 at 20:23 UTC
c0a4b95d6474c72799cd8af4421ce922654c850e
4 files changed
+10
-21
vllm/config/compilation.py
+3
-7
@@ -251,13 +251,6 @@ class CompilationConfig:
251
disabled when running with Inductor: mode>=VLLM_COMPILE and use_inductor=True.
252
Inductor generates (fused) Triton kernels for disabled custom ops."""
253
splitting_ops: list[str] | None = None
254
-
255
- """
256
- Provide control over whether to compile the multimodal encoder
257
- such as Qwen2_5_vl
258
- """
259
- compile_mm_encoder: bool = True
260
-
254
"""A list of ops to exclude from cudagraphs, used in piecewise compilation.
255
256
The behavior depends on use_inductor_graph_partition:
@@ -275,6 +268,9 @@ class CompilationConfig:
268
269
If None, defaults to attention ops for piecewise cudagraphs.
270
If empty list [], no ops are excluded (suitable for full cudagraphs)."""
271
+ compile_mm_encoder: bool = True
272
+ """Whether or not to compile the multimodal encoder.
273
+ Currently, this only works for `Qwen2_5_vl`."""
274
275
# Inductor capture
276
use_inductor: bool | None = None
vllm/model_executor/models/qwen2_5_vl.py
+1
-3
@@ -67,9 +67,7 @@ from vllm.model_executor.layers.linear import (
67
from vllm.model_executor.layers.quantization import QuantizationConfig
68
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
69
from vllm.model_executor.models.module_mapping import MultiModelKeys
70
-from vllm.model_executor.models.transformers.utils import (
71
- should_torch_compile_mm_vit,
72
-)
70
+from vllm.model_executor.models.vision import should_torch_compile_mm_vit
71
from vllm.multimodal import MULTIMODAL_REGISTRY
72
from vllm.multimodal.evs import (
73
compute_mrope_for_media,
vllm/model_executor/models/transformers/utils.py
-11
@@ -205,14 +205,3 @@ def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
205
# Dynamic rope scaling is not compatible with torch.compile
206
rope_scaling: dict = getattr(text_config, "rope_scaling", None) or {}
207
return rope_scaling.get("rope_type") != "dynamic"
208
-
209
-
210
-def should_torch_compile_mm_vit(vllm_config: "VllmConfig") -> bool:
211
- """
212
- Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
213
-
214
- Defaults to `True` but is disabled in the following situations:
215
-
216
- - The model uses dynamic rope scaling.
217
- """
218
- return vllm_config.compilation_config.compile_mm_encoder
vllm/model_executor/models/vision.py
+6
@@ -11,6 +11,7 @@ import torch
11
from transformers import PretrainedConfig
12
13
from vllm.attention.backends.registry import _Backend
14
+from vllm.config import VllmConfig
15
from vllm.distributed import (
16
get_tensor_model_parallel_rank,
17
get_tensor_model_parallel_world_size,
@@ -100,6 +101,11 @@ def get_vit_attn_backend(
101
return current_platform.get_vit_attn_backend(head_size, dtype)
102
103
104
+def should_torch_compile_mm_vit(vllm_config: VllmConfig) -> bool:
105
+ """Callable to be passed to `@support_torch_compile`'s `enable_if` argument."""
106
+ return vllm_config.compilation_config.compile_mm_encoder
107
+
108
+
109
VisionFeatureSelectStrategyStr = Literal["class", "default", "full"]
110
111
VisionFeatureSelectStrategy: TypeAlias = (