A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 01:00 UTC
Fix TURBOQUANT backend selection in cuda.py (#40060)
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Michael Goin committed
Apr 17, 2026 at 10:31 UTC
1174723eba176fcc68673a5a0a2c0090416aada8
2 files changed
+4
-5
docs/design/attention_backends.md
+2
@@ -106,6 +106,7 @@ Priority is **1 = highest** (tried first).
106
| 2 | `FLASH_ATTN` |
107
| 3 | `TRITON_ATTN` |
108
| 4 | `FLEX_ATTENTION` |
109
+| 5 | `TURBOQUANT` |
110
111
**Ampere/Hopper (SM 8.x-9.x):**
112
@@ -115,6 +116,7 @@ Priority is **1 = highest** (tried first).
116
| 2 | `FLASHINFER` |
117
| 3 | `TRITON_ATTN` |
118
| 4 | `FLEX_ATTENTION` |
119
+| 5 | `TURBOQUANT` |
120
121
### MLA Attention (DeepSeek-style)
122
vllm/platforms/cuda.py
+2
-5
@@ -131,6 +131,7 @@ def _get_backend_priorities(
131
AttentionBackendEnum.FLASH_ATTN,
132
AttentionBackendEnum.TRITON_ATTN,
133
AttentionBackendEnum.FLEX_ATTENTION,
134
+ AttentionBackendEnum.TURBOQUANT,
135
]
136
else:
137
return [
@@ -138,6 +139,7 @@ def _get_backend_priorities(
139
AttentionBackendEnum.FLASHINFER,
140
AttentionBackendEnum.TRITON_ATTN,
141
AttentionBackendEnum.FLEX_ATTENTION,
142
+ AttentionBackendEnum.TURBOQUANT,
143
]
144
145
@@ -255,11 +257,6 @@ class CudaPlatformBase(Platform):
257
valid_backends_priorities = []
258
invalid_reasons: dict[AttentionBackendEnum, tuple[int, list[str]]] = {}
259
258
- # TurboQuant KV cache: route directly to TQ backend
259
- kv_cache_dtype = attn_selector_config.kv_cache_dtype
260
- if kv_cache_dtype is not None and kv_cache_dtype.startswith("turboquant_"):
261
- return [(AttentionBackendEnum.TURBOQUANT, 0)], {}
262
-
260
backend_priorities = _get_backend_priorities(
261
attn_selector_config.use_mla,
262
device_capability,