[Bugfix] Fix Llama4ForCausalLM initialization test failure (#46341)
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com>
liuzhenwei committed
Jun 22, 2026 at 16:40 UTC
435f82d61a1eddb84854ca59a008a8e4d97ab439
1 file changed
+7
-1
tests/models/utils.py
+7
-1
@@ -507,7 +507,13 @@ def dummy_hf_overrides(
507
# Only set MoE related config when the model has MoE layers.
508
# Otherwise all models detected as MoE by _get_transformers_backend_cls.
509
if model_arch_config.num_experts > 0:
510
- num_experts_per_tok = 1 if model_arch == "Llama4ForConditionalGeneration" else 2
510
+ num_experts_per_tok = 2
511
+ if model_arch in (
512
+ "Llama4ForConditionalGeneration",
513
+ "Llama4ForCausalLM",
514
+ "EagleLlama4ForCausalLM",
515
+ ):
516
+ num_experts_per_tok = 1
517
update_dict.update(
518
{
519
"num_experts": num_experts,