[Quant] Enable modelopt_mixed on Turing (SM75) (#45375)
Signed-off-by: Mike G <180722391+mikekg@users.noreply.github.com>
Mike G committed
Jun 22, 2026 at 19:30 UTC
56e57975112b7931124dac81f4edae458a26f7bb
3 files changed
+15
-9
docs/design/attention_backends.md
+1
-1
@@ -160,7 +160,7 @@ Priority is **1 = highest** (tried first).
160
| Backend | Version | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | MM Prefix | DCP | Attention Types | Compute Cap. |
161
| ------- | ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | --------- | --- | --------------- | ------------ |
162
| `CPU_ATTN` | | fp16, bf16, fp32 | `auto`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 112, 128, 160, 192, 224, 256, 512 | ❌ | ❌ | ❌ | ❌ | All | N/A |
163
-| `FLASHINFER` | Native† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ❌ | ✅ | ❌ | ✅ | Decoder | 7.x-9.x |
163
+| `FLASHINFER` | Native† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ❌ | ✅ | ❌ | ✅ | Decoder | 8.x-9.x |
164
| `FLASHINFER` | TRTLLM† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `nvfp4` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ✅ | ✅ | ❌ | ✅ | Decoder | 10.x |
165
| `FLASH_ATTN` | FA2* | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ✅ | ❌ | ✅ | All | ≥8.0 |
166
| `FLASH_ATTN` | FA3* | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | 9.x |
vllm/model_executor/layers/quantization/modelopt.py
+8
-7
@@ -2299,13 +2299,14 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
2299
2300
@classmethod
2301
def get_min_capability(cls) -> int:
2302
- # Ampere (SM80/SM86): NVFP4 routed experts run via Marlin W4A16, and FP8
2303
- # weight-only dense layers run via MarlinFP8 (W8A16, compute in
2304
- # bf16/fp16). FP8 MoE, if present, also routes to Marlin because
2305
- # TritonExperts gates its FP8 schemes behind supports_fp8() (cc>=89).
2306
- # None of these paths require native FP8 tensor cores, so SM80 is
2307
- # sufficient.
2308
- return 80
2302
+ # Turing and up (SM75+): NVFP4 routed experts run via Marlin W4A16
2303
+ # (SM75+), FP8 weight-only dense via MarlinFP8 (cc>=7.5), and FP8 MoE,
2304
+ # if present, via Marlin (TritonExperts gates its FP8 schemes behind
2305
+ # supports_fp8(), cc>=89). None of these paths require native FP8 tensor
2306
+ # cores, so SM75 is sufficient. Validated end-to-end on a Tesla T4
2307
+ # (SM75) and A100 (SM80). Pairs with the FlashInfer attention SM80
2308
+ # lower bound so SM75 auto-selects a supported attention backend.
2309
+ return 75
2310
2311
@classmethod
2312
def override_quantization_method(
vllm/v1/attention/backends/flashinfer.py
+6
-1
@@ -424,7 +424,12 @@ class FlashInferBackend(AttentionBackend):
424
425
@classmethod
426
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
427
- return capability >= DeviceCapability(7, 5) and capability <= DeviceCapability(
427
+ # FlashInfer supports SM75+, but is currently broken on SM75 (Turing):
428
+ # https://github.com/flashinfer-ai/flashinfer/issues/3620 (fix:
429
+ # https://github.com/flashinfer-ai/flashinfer/pull/3621). Temporarily
430
+ # raise the floor to SM80 so it is not auto-selected on SM75 until
431
+ # that fix lands; revert to DeviceCapability(7, 5) once it does.
432
+ return capability >= DeviceCapability(8, 0) and capability <= DeviceCapability(
433
12, 1
434
)
435