A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model][Bugfix] fix ernie45 vl run failed from shared experts optimization (#26885)
Signed-off-by: wangyafeng <wangyafeng@baidu.com>
CSWYF3634076 committed
Oct 16, 2025 at 18:37 UTC
e51928793e108a3a10ba6e02ba907f8e482ddc40
1 file changed
+22
-5
vllm/model_executor/models/ernie45_vl_moe.py
+22
-5
@@ -341,7 +341,10 @@ class Ernie4_5_VLMoeMoE(nn.Module):
341
# text and vision modals input
342
visual_token_mask = visual_token_mask.repeat(1, self.hidden_size).bool()
343
text_token_mask = ~visual_token_mask
344
- final_hidden_states = torch.zeros_like(hidden_states)
344
+ final_experts_hidden_states = torch.zeros_like(hidden_states)
345
+ final_shared_ouput = (
346
+ torch.zeros_like(hidden_states) if self.has_shared_experts else None
347
+ )
348
349
text_hidden_states = hidden_states[text_token_mask].reshape(
350
-1, self.hidden_size
@@ -353,16 +356,26 @@ class Ernie4_5_VLMoeMoE(nn.Module):
356
text_router_logits, _ = self.text_experts_gate(
357
text_hidden_states.to(dtype=torch.float32)
358
)
356
- final_hidden_states[text_token_mask] = self.text_experts(
359
+ text_shared_ouput, text_experts_output = self.text_experts(
360
hidden_states=text_hidden_states, router_logits=text_router_logits
358
- ).flatten()
361
+ )
362
+ final_experts_hidden_states[text_token_mask] = text_experts_output.flatten()
363
+ if self.has_shared_experts:
364
+ final_shared_ouput[text_token_mask] = text_shared_ouput.flatten()
365
366
vision_router_logits, _ = self.vision_experts_gate(
367
vision_hidden_states.to(dtype=torch.float32)
368
)
363
- final_hidden_states[visual_token_mask] = self.vision_experts(
369
+ vision_shared_ouput, vision_experts_output = self.vision_experts(
370
hidden_states=vision_hidden_states, router_logits=vision_router_logits
365
- ).flatten()
371
+ )
372
+ final_experts_hidden_states[visual_token_mask] = (
373
+ vision_experts_output.flatten()
374
+ )
375
+ if self.has_shared_experts:
376
+ final_shared_ouput[visual_token_mask] = vision_shared_ouput.flatten()
377
+
378
+ final_hidden_states = (final_shared_ouput, final_experts_hidden_states)
379
else:
380
# only text modal input
381
text_router_logits, _ = self.text_experts_gate(
@@ -374,7 +387,11 @@ class Ernie4_5_VLMoeMoE(nn.Module):
387
)
388
389
if self.has_shared_experts:
390
+ # for shared_experts model
391
final_hidden_states = final_hidden_states[0] + final_hidden_states[1]
392
+ else:
393
+ # for not shared_experts model
394
+ final_hidden_states = final_hidden_states[1]
395
396
if self.tp_size > 1:
397
final_hidden_states = (