A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 17:30 UTC
[Misc] use model arch converter for bidi models identification (#40701)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Apr 23, 2026 at 21:42 UTC
da1e7311cad640e008284c2fb1754f5f692259c8
3 files changed
+28
-15
vllm/config/model.py
+2
-15
@@ -1197,22 +1197,9 @@ class ModelConfig:
1197
def is_deepseek_mla(self) -> bool:
1198
return self.model_arch_config.is_deepseek_mla
1199
1200
- @cached_property
1200
+ @property
1201
def is_mm_prefix_lm(self) -> bool:
1202
- """Whether to use bidirectional attention for mm positions."""
1203
- if hasattr(self.hf_config, "is_mm_prefix_lm"):
1204
- return bool(self.hf_config.is_mm_prefix_lm)
1205
- # fallback to list of known models
1206
- MM_PREFIX_LM_MODELS = (
1207
- "bagel",
1208
- "gemma3",
1209
- "molmo2",
1210
- "paligemma",
1211
- "umm",
1212
- )
1213
- if not hasattr(self.hf_config, "model_type"):
1214
- return False
1215
- return self.hf_config.model_type in MM_PREFIX_LM_MODELS
1202
+ return self.model_arch_config.is_mm_prefix_lm
1203
1204
def get_head_size(self) -> int:
1205
return self.model_arch_config.head_size
vllm/config/model_arch.py
+3
@@ -53,5 +53,8 @@ class ModelArchitectureConfig:
53
is_deepseek_mla: bool
54
"""Whether the model is a DeepSeek MLA model."""
55
56
+ is_mm_prefix_lm: bool
57
+ """Whether the model uses image bidirectional attention."""
58
+
59
derived_max_model_len_and_key: tuple[float, str | None]
60
"""Derived maximum model length and key from the hf config."""
vllm/transformers_utils/model_arch_config_convertor.py
+23
@@ -250,6 +250,22 @@ class ModelArchConfigConvertorBase:
250
)
251
return False
252
253
+ def is_mm_prefix_lm(self) -> bool:
254
+ """Whether to use bidirectional attention for mm positions."""
255
+ if hasattr(self.hf_config, "is_mm_prefix_lm"):
256
+ return bool(self.hf_config.is_mm_prefix_lm)
257
+ # fallback to list of known models
258
+ MM_PREFIX_LM_MODELS = (
259
+ "bagel",
260
+ "gemma3",
261
+ "molmo2",
262
+ "paligemma",
263
+ "umm",
264
+ )
265
+ if not hasattr(self.hf_config, "model_type"):
266
+ return False
267
+ return self.hf_config.model_type in MM_PREFIX_LM_MODELS
268
+
269
def derive_max_model_len_and_key(self) -> tuple[float, str | None]:
270
derived_max_model_len = float("inf")
271
possible_keys = [
@@ -299,6 +315,7 @@ class ModelArchConfigConvertorBase:
315
num_experts=self.get_num_experts(),
316
quantization_config=self.get_quantization_config(),
317
is_deepseek_mla=self.is_deepseek_mla(),
318
+ is_mm_prefix_lm=self.is_mm_prefix_lm(),
319
derived_max_model_len_and_key=self.derive_max_model_len_and_key(),
320
)
321
@@ -451,6 +468,12 @@ class LongCatFlashMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
468
469
470
class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase):
471
+ def is_mm_prefix_lm(self) -> bool:
472
+ return (
473
+ getattr(self.hf_text_config, "use_bidirectional_attention", None)
474
+ == "vision"
475
+ )
476
+
477
def get_head_size(self) -> int:
478
# Gemma4 uses dual head dimensions: head_dim (sliding attention)
479
# and global_head_dim (full attention). Return the largest so