A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 21:00 UTC
[MoE] Add RoutingMethodType.Simulated to TRT-LLM FP8/NVFP4 kernel allowlists (#38329)
Signed-off-by: Jaewon Lee <jaewon@meta.com>
Jaewon committed
Mar 29, 2026 at 22:53 UTC
d816834c1aa61da8e8703c6e8c368ad22fe76e13
2 files changed
+21
-8
vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py
+11
-4
@@ -256,13 +256,18 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
256
) -> bool:
257
"""
258
The FlashInfer TRTLLM FP8 kernel expects bfloat16 router_logits by default.
259
- Only DeepSeekV3 routing supports float32 router_logits (which is converted
260
- internally in the kernel).
259
+ DeepSeekV3 routing supports float32 router_logits (converted internally).
260
+ Simulated routing generates synthetic decisions and is agnostic to dtype.
261
"""
262
if router_logits_dtype == torch.float32:
263
- # Only DeepSeekV3 routing handles float32 logits
263
+ # DeepSeekV3 routing handles float32 logits internally.
264
+ # Simulated routing generates synthetic decisions, so the
265
+ # kernel doesn't care about the actual logits dtype.
266
# https://github.com/flashinfer-ai/flashinfer/issues/2469
265
- return routing_method == RoutingMethodType.DeepSeekV3
267
+ return routing_method in (
268
+ RoutingMethodType.DeepSeekV3,
269
+ RoutingMethodType.Simulated,
270
+ )
271
return True
272
273
@staticmethod
@@ -288,12 +293,14 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
293
# NOTE(rob): potentially allow others here. This is a conservative list.
294
return routing_method in [
295
RoutingMethodType.DeepSeekV3,
296
+ RoutingMethodType.Simulated,
297
]
298
elif (weight_key, activation_key) == (kFp8StaticTensorSym, kFp8StaticTensorSym):
299
# NOTE(dbari): as above, potentially allow others here.
300
return routing_method in [
301
RoutingMethodType.DeepSeekV3,
302
RoutingMethodType.Llama4,
303
+ RoutingMethodType.Simulated,
304
]
305
else:
306
raise ValueError("Unsupported quantization scheme.")
vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py
+10
-4
@@ -255,6 +255,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
255
RoutingMethodType.Renormalize,
256
RoutingMethodType.RenormalizeNaive,
257
RoutingMethodType.Llama4,
258
+ RoutingMethodType.Simulated,
259
]
260
261
@staticmethod
@@ -264,13 +265,18 @@ class TrtLlmNvFp4ExpertsMonolithic(
265
) -> bool:
266
"""
267
The FlashInfer TRTLLM NvFp4 kernel expects bfloat16 router_logits by default.
267
- Only DeepSeekV3 routing supports float32 router_logits (which is converted
268
- internally in the kernel).
268
+ DeepSeekV3 routing supports float32 router_logits (converted internally).
269
+ Simulated routing generates synthetic decisions and is agnostic to dtype.
270
"""
271
if router_logits_dtype == torch.float32:
271
- # Only DeepSeekV3 routing handles float32 logits
272
+ # DeepSeekV3 routing handles float32 logits internally.
273
+ # Simulated routing generates synthetic decisions, so the
274
+ # kernel doesn't care about the actual logits dtype.
275
# https://github.com/flashinfer-ai/flashinfer/issues/2469
273
- return routing_method == RoutingMethodType.DeepSeekV3
276
+ return routing_method in (
277
+ RoutingMethodType.DeepSeekV3,
278
+ RoutingMethodType.Simulated,
279
+ )
280
return True
281
282
def apply(