A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 10:00 UTC
[Bugfix] Fix Sparse24 Compressed Tensors models (#33446)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
Kyle Sayers committed
Feb 12, 2026 at 02:15 UTC
e9cd6911321f7671de218d0c778c5400d7f1d1a6
3 files changed
+17
-15
csrc/sparse/cutlass/sparse_scaled_mm_entry.cu
+3
-3
@@ -6,11 +6,11 @@
6
#include "cutlass_extensions/common.hpp"
7
8
bool cutlass_sparse_scaled_mm_supported(int64_t cuda_device_capability) {
9
- // sparse CUTLASS kernels need at least
9
+ // sparse CUTLASS kernels need exactly hopper and are not forward compatible
10
// CUDA 12.2 and SM90 (Hopper)
11
12
#if defined CUDA_VERSION
13
- return CUDA_VERSION >= 12020 && cuda_device_capability >= 90;
13
+ return CUDA_VERSION >= 12020 && cuda_device_capability == 90;
14
#endif
15
16
return false;
@@ -98,7 +98,7 @@ std::vector<torch::Tensor> cutlass_sparse_compress(torch::Tensor const& a) {
98
99
TORCH_CHECK_NOT_IMPLEMENTED(
100
false,
101
- "No compiled cutlass_sparse_compress for a compute capability less than "
101
+ "No compiled cutlass_sparse_compress for a compute capability equal to "
102
"CUDA device capability: ",
103
version_num);
104
}
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+13
-12
@@ -207,18 +207,19 @@ class CompressedTensorsConfig(QuantizationConfig):
207
# because Attention quantization on its own is not supported by vLLM.
208
# It is coupled with KV-cache quantization, and if scales are present in the
209
# checkpoint, they will be used properly.
210
- grps_without_attn_quant = {}
211
- for k, v in config["config_groups"].items():
212
- # e.g. LlamaAttention, Qwen3Attention, etc.
213
- if len(v["targets"]) == 1 and v["targets"][0].endswith("Attention"):
214
- logger.warning(
215
- "Skipping CompressedTensors config group for %s. Attention quant "
216
- "is coupled with KV-cache quantization in vLLM.",
217
- v["targets"][0],
218
- )
219
- continue
220
- grps_without_attn_quant[k] = v
221
- config["config_groups"] = grps_without_attn_quant
210
+ if "config_groups" in config:
211
+ grps_without_attn_quant = {}
212
+ for k, v in config["config_groups"].items():
213
+ # e.g. LlamaAttention, Qwen3Attention, etc.
214
+ if len(v["targets"]) == 1 and v["targets"][0].endswith("Attention"):
215
+ logger.warning(
216
+ "Skipping CompressedTensors config group for %s. Attention "
217
+ "quant is coupled with KV-cache quantization in vLLM.",
218
+ v["targets"][0],
219
+ )
220
+ continue
221
+ grps_without_attn_quant[k] = v
222
+ config["config_groups"] = grps_without_attn_quant
223
224
ignore: list[str] = cast(list[str], config.get("ignore", []))
225
quant_format = cast(str, config.get("format"))
vllm/model_executor/model_loader/weight_utils.py
+1
@@ -261,6 +261,7 @@ def get_quant_config(
261
if (
262
hf_quant_config is not None
263
and hf_quant_config.get("quant_method") == "compressed-tensors"
264
+ and "config_groups" in hf_quant_config
265
):
266
if hf_text_config is not None:
267
n_heads = getattr(hf_text_config, "num_attention_heads", None)