A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 07:30 UTC
[Bugfix] Exclude `language_model_only` key from MM AOT compile hash but include in model one (#34508)
Signed-off-by: Roger Wang <hey@rogerw.io>
Roger Wang committed
Feb 13, 2026 at 05:59 UTC
1dae7b7843062f3468485653779d43ef96c7245c
3 files changed
+24
-1
tests/config/test_multimodal_config.py
+18
@@ -3,6 +3,7 @@
3
4
import pytest
5
6
+from vllm.config.model import ModelConfig
7
from vllm.config.multimodal import MultiModalConfig
8
from vllm.v1.attention.backends.registry import AttentionBackendEnum
9
@@ -23,3 +24,20 @@ def test_mm_encoder_attn_backend_hash_updates():
24
mm_encoder_attn_backend=AttentionBackendEnum.FLASH_ATTN
25
).compute_hash()
26
assert base_hash != overridden_hash
27
+
28
+
29
+def test_language_model_only_does_not_affect_mm_hash():
30
+ """language_model_only does not affect the ViT computation graph,
31
+ so it should not change the multimodal config hash."""
32
+ base_hash = MultiModalConfig().compute_hash()
33
+ lm_only_hash = MultiModalConfig(language_model_only=True).compute_hash()
34
+ assert base_hash == lm_only_hash
35
+
36
+
37
+def test_language_model_only_affects_model_hash():
38
+ """language_model_only affects the LM computation graph,
39
+ so it should change the model config hash."""
40
+ model = "llava-hf/llava-1.5-7b-hf"
41
+ base_hash = ModelConfig(model).compute_hash()
42
+ lm_only_hash = ModelConfig(model, language_model_only=True).compute_hash()
43
+ assert base_hash != lm_only_hash
vllm/config/model.py
+6
@@ -357,6 +357,12 @@ class ModelConfig:
357
from vllm.config.utils import get_hash_factors, hash_factors
358
359
factors = get_hash_factors(self, ignored_factors)
360
+
361
+ # NOTE: For some models (e.g, Qwen3-VL), whether the MM code path is enabled
362
+ # affects the computation graph of the language model, therefore we add it
363
+ # here early.
364
+ if self.multimodal_config:
365
+ factors["language_model_only"] = self.multimodal_config.language_model_only
366
return hash_factors(factors)
367
368
def _update_nested(
vllm/config/multimodal.py
-1
@@ -219,7 +219,6 @@ class MultiModalConfig:
219
the final hidden states.
220
"""
221
factors: list[Any] = [
222
- self.language_model_only,
222
self.mm_encoder_attn_backend.name
223
if self.mm_encoder_attn_backend is not None
224
else None,