A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix][Model] Fix Ernie4.5-VL hanging on long inputs (#24074)
Signed-off-by: wangyafeng <wangyafeng@baidu.com>
CSWYF3634076 committed
Sep 9, 2025 at 11:37 UTC
b6fbc15634731285b2907d2eab87c51f69f1a102
2 files changed
+18
-7
vllm/model_executor/models/ernie45_vl.py
+10
-4
@@ -66,8 +66,6 @@ from .vision import get_vit_attn_backend
66
67
logger = init_logger(__name__)
68
69
-_MAX_FRAMES_PER_VIDEO = 16
70
-
69
# === Vision Transformer === #
70
71
@@ -839,6 +837,15 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
837
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
838
return {"image": None, "video": None}
839
840
+ def get_mm_max_tokens_per_item(
841
+ self,
842
+ seq_len: int,
843
+ mm_counts: Mapping[str, int],
844
+ ) -> Mapping[str, int]:
845
+ max_image_tokens = self.get_max_image_tokens()
846
+ max_video_tokens = self.get_max_video_tokens(seq_len, mm_counts)
847
+ return {"image": max_image_tokens, "video": max_video_tokens}
848
+
849
def _get_vision_info(
850
self,
851
*,
@@ -964,8 +971,7 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
971
max_image_tokens = self.get_max_image_tokens() * max_images
972
max_total_frames = self._get_max_video_frames(seq_len -
973
max_image_tokens)
967
- max_frames_per_video = min(max_total_frames // max(max_videos, 1),
968
- _MAX_FRAMES_PER_VIDEO)
974
+ max_frames_per_video = max_total_frames // max(max_videos, 1)
975
976
return max(max_frames_per_video, 2)
977
vllm/model_executor/models/ernie45_vl_moe.py
+8
-3
@@ -287,8 +287,13 @@ class Ernie4_5_VLMoeMoE(nn.Module):
287
if self.has_shared_experts:
288
shared_output = self.shared_experts(hidden_states)
289
290
- if visual_token_mask is not None and visual_token_mask.any():
291
- # assert visual_token_mask.shape[0] != hidden_states.shape[0]
290
+ if visual_token_mask is not None and visual_token_mask.all():
291
+ # only vision modal input
292
+ router_logits, _ = self.vision_experts_gate(hidden_states)
293
+ final_hidden_states = self.vision_experts(
294
+ hidden_states=hidden_states, router_logits=router_logits)
295
+ elif visual_token_mask is not None and visual_token_mask.any():
296
+ # text and vision modals input
297
visual_token_mask = visual_token_mask.repeat(
298
1, self.hidden_size).bool()
299
text_token_mask = ~visual_token_mask
@@ -310,7 +315,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
315
hidden_states=vision_hidden_states,
316
router_logits=vision_router_logits).flatten()
317
else:
313
- # text modal input processing directly
318
+ # only text modal input
319
text_router_logits, _ = self.text_experts_gate(hidden_states)
320
321
final_hidden_states = self.text_experts(