A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 07:00 UTC
[Bugfix] Disable monolithic TRTLLM MoE for Renormalize routing (#37591) (#37605)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
Vadim Gimpelson committed
Mar 20, 2026 at 23:19 UTC
4f16ebbbd35e21b7d98173c406f51853a72a7157
5 files changed
+42
-5
.buildkite/test_areas/lm_eval.yaml
+16
@@ -45,6 +45,22 @@ steps:
45
commands:
46
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt
47
48
+- label: LM Eval Qwen3.5 Models (B200)
49
+ timeout_in_minutes: 120
50
+ device: b200
51
+ optional: true
52
+ num_devices: 2
53
+ source_file_dependencies:
54
+ - vllm/model_executor/models/qwen3_5.py
55
+ - vllm/model_executor/models/qwen3_5_mtp.py
56
+ - vllm/transformers_utils/configs/qwen3_5.py
57
+ - vllm/transformers_utils/configs/qwen3_5_moe.py
58
+ - vllm/model_executor/models/qwen3_next.py
59
+ - vllm/model_executor/models/qwen3_next_mtp.py
60
+ - vllm/model_executor/layers/fla/ops/
61
+ commands:
62
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
63
+
64
- label: LM Eval Large Models (H200)
65
timeout_in_minutes: 60
66
device: h200
tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "Qwen/Qwen3.5-35B-A3B"
2
+accuracy_threshold: 0.86
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: >-
6
+ --max-model-len 4096
7
+ --data-parallel-size 2
8
+ --enable-expert-parallel
tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-FP8-DEP2.yaml
new
+9
@@ -0,0 +1,9 @@
1
+model_name: "Qwen/Qwen3.5-35B-A3B-FP8"
2
+accuracy_threshold: 0.86
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: >-
6
+ --max-model-len 4096
7
+ --data-parallel-size 2
8
+ --enable-expert-parallel
9
+ --kv-cache-dtype fp8
tests/evals/gsm8k/configs/models-qwen35-blackwell.txt
new
+1
@@ -0,0 +1 @@
1
+Qwen3.5-35B-A3B-DEP2.yaml
vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py
+8
-5
@@ -269,9 +269,16 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
269
weight_key: QuantKey | None,
270
activation_key: QuantKey | None,
271
) -> bool:
272
- """Monolithic kernels need to express router support."""
272
+ """Monolithic kernels need to express router support.
273
+ Renormalize/RenormalizeNaive are excluded: the monolithic kernel's
274
+ internal routing for these methods produces output uncorrelated
275
+ with the modular kernel's output and with Triton kernel's output
276
+ for Qwen3.5-35B-A3B-FP8.
277
+ See: https://github.com/vllm-project/vllm/issues/37591
278
+ """
279
# NOTE(dbari): TopK routing could also be enabled, but need to validate models
280
# NOTE(dbari): Default is not implemented and should not be enabled until it is
281
+
282
if (weight_key, activation_key) in [
283
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
284
(kMxfp8Static, kMxfp8Dynamic),
@@ -279,16 +286,12 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
286
# NOTE(rob): potentially allow others here. This is a conservative list.
287
return routing_method in [
288
RoutingMethodType.DeepSeekV3,
282
- RoutingMethodType.Renormalize,
283
- RoutingMethodType.RenormalizeNaive,
289
]
290
elif (weight_key, activation_key) == (kFp8StaticTensorSym, kFp8StaticTensorSym):
291
# NOTE(dbari): as above, potentially allow others here.
292
return routing_method in [
293
RoutingMethodType.DeepSeekV3,
294
RoutingMethodType.Llama4,
290
- RoutingMethodType.Renormalize,
291
- RoutingMethodType.RenormalizeNaive,
295
]
296
else:
297
raise ValueError("Unsupported quantization scheme.")