[Perf] Improve `--linear-backend` filtering (#48735)
Signed-off-by: Aleksandr Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Aleksandr Skliar <askliar@nvidia.com>
Andrii Skliar committed
Aug 8, 2026 at 00:15 UTC
a801e71cb86ecd45cd8cb8f7d1bc08390f2bd68f
1 file changed
+37
-48
vllm/model_executor/kernels/linear/__init__.py
+37
-48
@@ -328,6 +328,37 @@ def _filter_kernels_by_backend(
328
return [k for k in kernels if k in backend_kernels]
329
330
331
+def _resolve_backend_kernels(
332
+ kernels: list[type],
333
+ layer_desc: str,
334
+) -> list[type]:
335
+ """Apply --linear-backend filtering to one layer type's kernel list.
336
+
337
+ When the requested backend has no kernel for this layer type, fall back
338
+ to the unfiltered list (with a WARNING log) instead of failing engine
339
+ startup: an explicitly requested backend usually provides kernels for a
340
+ single quantization scheme, while a model can contain several linear
341
+ layer types (e.g. NVFP4 MoE projections next to FP8 attention
342
+ projections).
343
+ """
344
+ linear_backend = _get_linear_backend()
345
+ if linear_backend == "auto":
346
+ return kernels
347
+ filtered = _filter_kernels_by_backend(linear_backend, kernels)
348
+ if not filtered:
349
+ logger.warning_once(
350
+ "--linear-backend=%s was requested, but no %s kernel exists for "
351
+ "%s layers; falling back to normal kernel selection for this "
352
+ "layer.",
353
+ linear_backend,
354
+ linear_backend,
355
+ layer_desc,
356
+ scope="global",
357
+ )
358
+ return kernels
359
+ return filtered
360
+
361
+
362
# in priority/performance order (when available)
363
_POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]] = {
364
PlatformEnum.CPU: [ZentorchInt8ScaledMMLinearKernel, CPUInt8ScaledMMLinearKernel],
@@ -467,10 +498,8 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
498
_POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
499
PlatformEnum.CUDA: [
500
FlashInferCuteDslNvFp4LinearKernel,
470
- # FlashInferB12xNvFp4LinearKernel excluded from auto-selection until
471
- # upstream CUTLASS SM121 MMA op guard is resolved; use
472
- # --linear-backend flashinfer_b12x to opt in explicitly.
501
FlashInferCutlassNvFp4LinearKernel,
502
+ FlashInferB12xNvFp4LinearKernel,
503
CutlassNvFp4LinearKernel,
504
MarlinNvFp4LinearKernel,
505
FlashInferTrtllmNvFp4LinearKernel,
@@ -588,15 +617,7 @@ def choose_scaled_mm_linear_kernel(
617
platform_kernels = possible_kernels.get(current_platform._enum, [])
618
619
# Apply --linear-backend filtering when set.
591
- linear_backend = _get_linear_backend()
592
- if linear_backend != "auto":
593
- filtered = _filter_kernels_by_backend(linear_backend, platform_kernels)
594
- if not filtered:
595
- raise ValueError(
596
- f"--linear-backend={linear_backend} was requested but no "
597
- f"'{linear_backend}' kernel exists for this layer type."
598
- )
599
- platform_kernels = filtered
620
+ platform_kernels = _resolve_backend_kernels(platform_kernels, "scaled-mm")
621
622
for kernel in platform_kernels:
623
is_supported_and_can_implement, failure_reason = (
@@ -752,15 +773,7 @@ def choose_mp_linear_kernel(
773
platform_kernels = _POSSIBLE_KERNELS.get(current_platform._enum, [])
774
775
# Apply --linear-backend filtering when set.
755
- linear_backend = _get_linear_backend()
756
- if linear_backend != "auto":
757
- filtered = _filter_kernels_by_backend(linear_backend, platform_kernels)
758
- if not filtered:
759
- raise ValueError(
760
- f"--linear-backend={linear_backend} was requested but no "
761
- f"'{linear_backend}' kernel exists for mixed-precision layers."
762
- )
763
- platform_kernels = filtered
776
+ platform_kernels = _resolve_backend_kernels(platform_kernels, "mixed-precision")
777
778
failure_reasons = []
779
for kernel in platform_kernels:
@@ -803,15 +816,7 @@ def init_mxfp8_linear_kernel() -> Mxfp8LinearKernel:
816
possible = list(_POSSIBLE_MXFP8_KERNELS.get(platform, []))
817
818
# Apply --linear-backend filtering when set.
806
- linear_backend = _get_linear_backend()
807
- if linear_backend != "auto":
808
- filtered = _filter_kernels_by_backend(linear_backend, possible)
809
- if not filtered:
810
- raise ValueError(
811
- f"--linear-backend={linear_backend} was requested but no "
812
- f"'{linear_backend}' kernel exists for MXFP8 layers."
813
- )
814
- possible = filtered
819
+ possible = _resolve_backend_kernels(possible, "MXFP8")
820
821
failure_reasons = []
822
for kernel_cls in possible:
@@ -849,20 +854,11 @@ def init_mxfp4_linear_kernel(
854
activation_quant_key=activation_quant_key,
855
)
856
852
- linear_backend = _get_linear_backend()
853
-
857
platform = current_platform._enum
858
possible = list(_POSSIBLE_MXFP4_KERNELS.get(platform, []))
859
860
# Apply --linear-backend filtering when set.
858
- if linear_backend != "auto":
859
- filtered = _filter_kernels_by_backend(linear_backend, possible)
860
- if not filtered:
861
- raise ValueError(
862
- f"--linear-backend={linear_backend} was requested but no "
863
- f"'{linear_backend}' kernel exists for MXFP4 layers."
864
- )
865
- possible = filtered
861
+ possible = _resolve_backend_kernels(possible, "MXFP4")
862
863
failure_reasons = []
864
for kernel_cls in possible:
@@ -1042,14 +1038,7 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
1038
possible = [kernel for kernel in possible if kernel in a16_kernels]
1039
1040
# Apply --linear-backend filtering when set.
1045
- if linear_backend != "auto":
1046
- filtered = _filter_kernels_by_backend(linear_backend, possible)
1047
- if not filtered:
1048
- raise ValueError(
1049
- f"--linear-backend={linear_backend} was requested but no "
1050
- f"'{linear_backend}' kernel exists for NVFP4 layers."
1051
- )
1052
- possible = filtered
1041
+ possible = _resolve_backend_kernels(possible, "NVFP4")
1042
1043
failure_reasons = []
1044
for kernel_cls in possible: