A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 16:00 UTC
[Fix] Add SM check to flashinfer MOE backend (#29144)
Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
jiahanc committed
Nov 22, 2025 at 16:39 UTC
5f96c00c557fc68c352d1b9bb1d6f9c9bb9f133d
1 file changed
+10
vllm/model_executor/layers/quantization/utils/flashinfer_utils.py
+10
@@ -282,6 +282,16 @@ def get_flashinfer_moe_backend() -> FlashinferMoeBackend:
282
283
flashinfer_moe_backend = envs.VLLM_FLASHINFER_MOE_BACKEND
284
if flashinfer_moe_backend in backend_map:
285
+ if (
286
+ flashinfer_moe_backend == "latency"
287
+ and not current_platform.is_device_capability(100)
288
+ ):
289
+ logger.info_once(
290
+ "Flashinfer TRTLLM MOE backend is only supported on "
291
+ "SM100 and later, using CUTLASS backend instead",
292
+ scope="local",
293
+ )
294
+ return FlashinferMoeBackend.CUTLASS
295
return backend_map[flashinfer_moe_backend]
296
elif current_platform.is_device_capability(90):
297
return FlashinferMoeBackend.CUTLASS