[Quantization] Enable humming wNaM asymmetric quant (zero_point) with compressed-tensors (#46528)
HDCharles committed
Sep 23, 2026 at 10:23 UTC
9f07d023d07f73d806a67b8616a0a35258da2fc5
12 files changed
+164
-34
.buildkite/test_areas/lm_eval.yaml
+2
-2
@@ -571,7 +571,7 @@ steps:
571
- label: ":nvidia: (H100) LM Eval Humming Activation FP8/INT8 TEMPORARY"
572
key: lm-eval-humming-h100-act
573
optional: true
574
- timeout_in_minutes: 70
574
+ timeout_in_minutes: 90
575
device: h100
576
num_devices: 1
577
source_file_dependencies:
@@ -582,7 +582,7 @@ steps:
582
- vllm/model_executor/kernels/linear/
583
commands:
584
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
585
- # - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
585
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
586
587
- label: ":nvidia: (H100) LM Eval Humming DeepEP TEMPORARY"
588
key: lm-eval-humming-h100-deepep
tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml
+2
-2
@@ -1,5 +1,5 @@
1
-model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM"
2
-accuracy_threshold: 0.78
1
+model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-a4"
2
+accuracy_threshold: 0.70
3
num_questions: 1319
4
num_fewshot: 5
5
server_args: >-
tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM-asym"
2
+accuracy_threshold: 0.85
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: >-
6
+ --enforce-eager
7
+ --max-model-len 4096
8
+ --linear-backend humming
tests/evals/gsm8k/configs/humming/config-a100-act-int8-shard-0.txt
+1
@@ -1,3 +1,4 @@
1
gpt-oss-20b-humming.yaml
2
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
3
Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml
4
+Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml
tests/evals/gsm8k/configs/humming/config-act-int8.txt
+1
-1
@@ -4,4 +4,4 @@ Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
4
Qwen3-30B-A3B-AWQ-humming.yaml
5
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
6
Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml
7
-Qwen3-4B-mixed-quant-RTN-humming.yaml
7
+Qwen3-4B-mixed-quant-RTN-no-w4a4-humming.yaml
vllm/model_executor/kernels/linear/mixed_precision/humming.py
+3
@@ -31,11 +31,14 @@ class HummingLinearKernel(MPLinearKernel):
31
)
32
33
name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}
34
+ if self.w_zp_name is not None and hasattr(layer, self.w_zp_name):
35
+ name_map["zero_point"] = self.w_zp_name
36
group_size = self.config.group_size
37
quant_config = {
38
"quant_method": "humming",
39
"dtype": "int" + str(self.config.weight_type.size_bits),
40
"group_size": 0 if group_size == -1 else group_size,
41
+ "has_zero_point": self.config.zero_points,
42
}
43
44
if self.config.zero_points:
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+2
@@ -787,6 +787,7 @@ class CompressedTensorsConfig(QuantizationConfig):
787
num_bits=weight_quant.num_bits,
788
strategy=weight_quant.strategy,
789
group_size=weight_quant.group_size,
790
+ symmetric=weight_quant.symmetric,
791
input_quant=input_quant,
792
layer_name=layer_name,
793
quant_format=format,
@@ -799,6 +800,7 @@ class CompressedTensorsConfig(QuantizationConfig):
800
num_bits=weight_quant.num_bits,
801
strategy=weight_quant.strategy,
802
group_size=weight_quant.group_size,
803
+ symmetric=weight_quant.symmetric,
804
input_quant=input_quant,
805
layer_name=layer_name,
806
quant_format=format,
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+9
-1
@@ -42,7 +42,15 @@ WNA16_SUPPORTED_TYPES_MAP = {
42
7: scalar_types.uint7b64,
43
8: scalar_types.uint8b128,
44
}
45
-WNA16_ZP_SUPPORTED_TYPES_MAP = {4: scalar_types.uint4, 8: scalar_types.uint8}
45
+WNA16_ZP_SUPPORTED_TYPES_MAP = {
46
+ 2: scalar_types.uint2,
47
+ 3: scalar_types.uint3,
48
+ 4: scalar_types.uint4,
49
+ 5: scalar_types.uint5,
50
+ 6: scalar_types.uint6,
51
+ 7: scalar_types.uint7,
52
+ 8: scalar_types.uint8,
53
+}
54
WNA16_SUPPORTED_BITS = list(WNA16_SUPPORTED_TYPES_MAP.keys())
55
56
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py
+59
-14
@@ -1,11 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""Weight N-bit INT scheme with symmetric INT4 activation quant via Humming.
3
+"""Weight N-bit INT scheme with INT4 activation quant via Humming.
4
5
Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit)
6
with INT4 symmetric dynamic per-token/per-group input activation
7
-quantization. Static, per-tensor, and asymmetric activation quantization
8
-are not supported.
7
+quantization. Supports both symmetric and asymmetric weight quantization.
8
+Static, per-tensor, and asymmetric activation quantization are not supported.
9
"""
10
11
import math
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
28
)
29
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
30
WNA16_SUPPORTED_TYPES_MAP,
31
+ WNA16_ZP_SUPPORTED_TYPES_MAP,
32
)
33
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
34
marlin_repeat_scales_on_all_ranks,
@@ -36,6 +37,7 @@ from vllm.model_executor.parameter import (
37
BasevLLMParameter,
38
ChannelQuantScaleParameter,
39
GroupQuantScaleParameter,
40
+ PackedColumnParameter,
41
PackedvLLMParameter,
42
)
43
@@ -52,6 +54,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
54
num_bits: int,
55
strategy: str,
56
group_size: int | None = None,
57
+ symmetric: bool = True,
58
input_quant: QuantizationArgs | None = None,
59
layer_name: str | None = None,
60
quant_format: str = "pack-quantized",
@@ -60,6 +63,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
63
self.pack_factor = Fraction(32, num_bits)
64
self.strategy = strategy
65
self.group_size = -1 if group_size is None else group_size
66
+ self.symmetric = symmetric
67
self.input_quant = input_quant
68
self.layer_name = layer_name
69
self.quant_format = quant_format
@@ -69,7 +73,19 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
73
f"Unsupported num_bits = {num_bits} for WNA4Int; "
74
f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}"
75
)
72
- self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
76
+
77
+ if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP:
78
+ raise ValueError(
79
+ f"Asymmetric quantization not supported for "
80
+ f"num_bits = {num_bits}. Supported: "
81
+ f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}"
82
+ )
83
+
84
+ self.quant_type = (
85
+ WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits]
86
+ if not self.symmetric
87
+ else WNA16_SUPPORTED_TYPES_MAP[num_bits]
88
+ )
89
90
if input_quant is not None:
91
if not input_quant.symmetric:
@@ -143,7 +159,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
159
weight_type=self.quant_type,
160
act_type=params_dtype,
161
group_size=self.group_size,
146
- zero_points=False,
162
+ zero_points=not self.symmetric,
163
)
164
165
kernel_type = choose_mp_linear_kernel(mp_config)
@@ -151,12 +167,6 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
167
logger.info("Using %s for CompressedTensorsWNA4Int", kernel_type.__name__)
168
self._kernel_backends_being_used.add(kernel_type.__name__)
169
154
- self.kernel = kernel_type(
155
- mp_config,
156
- w_q_param_name="weight_packed",
157
- w_s_param_name="weight_scale",
158
- )
159
-
170
input_quant_config = self._build_input_quant_config()
171
if input_quant_config is not None:
172
layer._humming_input_quant_config = input_quant_config
@@ -167,10 +177,10 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
177
partition_scales = not marlin_repeat_scales_on_all_ranks(
178
self.group_size, row_parallel
179
)
170
- scales_size = input_size // group_size
180
+ scales_and_zp_size = input_size // group_size
181
if partition_scales:
182
assert input_size_per_partition % group_size == 0
173
- scales_size = input_size_per_partition // group_size
183
+ scales_and_zp_size = input_size_per_partition // group_size
184
185
packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
186
layer.register_parameter(
@@ -190,7 +200,7 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
200
)
201
202
scale_data = torch.empty(
193
- output_size_per_partition, scales_size, dtype=params_dtype
203
+ output_size_per_partition, scales_and_zp_size, dtype=params_dtype
204
)
205
if partition_scales:
206
weight_scale = GroupQuantScaleParameter(
@@ -207,6 +217,34 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
217
)
218
layer.register_parameter("weight_scale", weight_scale)
219
220
+ if not self.symmetric:
221
+ packed_output_dim = math.ceil(
222
+ output_size_per_partition * self.num_bits / 32
223
+ )
224
+ zeros_data = torch.zeros(
225
+ packed_output_dim,
226
+ scales_and_zp_size,
227
+ dtype=torch.int32,
228
+ )
229
+ if not partition_scales:
230
+ qzeros = PackedColumnParameter(
231
+ output_dim=0,
232
+ packed_dim=0,
233
+ packed_factor=self.pack_factor,
234
+ weight_loader=weight_loader,
235
+ data=zeros_data,
236
+ )
237
+ else:
238
+ qzeros = PackedvLLMParameter(
239
+ input_dim=1,
240
+ output_dim=0,
241
+ packed_dim=0,
242
+ packed_factor=self.pack_factor,
243
+ weight_loader=weight_loader,
244
+ data=zeros_data,
245
+ )
246
+ layer.register_parameter("weight_zero_point", qzeros)
247
+
248
layer.register_parameter(
249
"weight_shape",
250
BasevLLMParameter(
@@ -215,6 +253,13 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
253
),
254
)
255
256
+ self.kernel = kernel_type(
257
+ mp_config,
258
+ w_q_param_name="weight_packed",
259
+ w_s_param_name="weight_scale",
260
+ w_zp_param_name="weight_zero_point",
261
+ )
262
+
263
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
264
self.kernel.process_weights_after_loading(layer)
265
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py
+59
-14
@@ -1,11 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""Weight N-bit INT scheme with symmetric INT8 activation quant via Humming.
3
+"""Weight N-bit INT scheme with INT8 activation quant via Humming.
4
5
Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit)
6
with INT8 symmetric dynamic per-token/per-group input activation
7
-quantization. Static, per-tensor, and asymmetric activation quantization
8
-are not supported.
7
+quantization. Supports both symmetric and asymmetric weight quantization.
8
+Static, per-tensor, and asymmetric activation quantization are not supported.
9
"""
10
11
import math
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
28
)
29
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
30
WNA16_SUPPORTED_TYPES_MAP,
31
+ WNA16_ZP_SUPPORTED_TYPES_MAP,
32
)
33
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
34
marlin_repeat_scales_on_all_ranks,
@@ -36,6 +37,7 @@ from vllm.model_executor.parameter import (
37
BasevLLMParameter,
38
ChannelQuantScaleParameter,
39
GroupQuantScaleParameter,
40
+ PackedColumnParameter,
41
PackedvLLMParameter,
42
)
43
@@ -52,6 +54,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
54
num_bits: int,
55
strategy: str,
56
group_size: int | None = None,
57
+ symmetric: bool = True,
58
input_quant: QuantizationArgs | None = None,
59
layer_name: str | None = None,
60
quant_format: str = "pack-quantized",
@@ -60,6 +63,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
63
self.pack_factor = Fraction(32, num_bits)
64
self.strategy = strategy
65
self.group_size = -1 if group_size is None else group_size
66
+ self.symmetric = symmetric
67
self.input_quant = input_quant
68
self.layer_name = layer_name
69
self.quant_format = quant_format
@@ -69,7 +73,19 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
73
f"Unsupported num_bits = {num_bits} for WNA8Int; "
74
f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}"
75
)
72
- self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
76
+
77
+ if not self.symmetric and num_bits not in WNA16_ZP_SUPPORTED_TYPES_MAP:
78
+ raise ValueError(
79
+ f"Asymmetric quantization not supported for "
80
+ f"num_bits = {num_bits}. Supported: "
81
+ f"{list(WNA16_ZP_SUPPORTED_TYPES_MAP)}"
82
+ )
83
+
84
+ self.quant_type = (
85
+ WNA16_ZP_SUPPORTED_TYPES_MAP[num_bits]
86
+ if not self.symmetric
87
+ else WNA16_SUPPORTED_TYPES_MAP[num_bits]
88
+ )
89
90
if input_quant is not None:
91
if not input_quant.symmetric:
@@ -143,7 +159,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
159
weight_type=self.quant_type,
160
act_type=params_dtype,
161
group_size=self.group_size,
146
- zero_points=False,
162
+ zero_points=not self.symmetric,
163
)
164
165
kernel_type = choose_mp_linear_kernel(mp_config)
@@ -151,12 +167,6 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
167
logger.info("Using %s for CompressedTensorsWNA8Int", kernel_type.__name__)
168
self._kernel_backends_being_used.add(kernel_type.__name__)
169
154
- self.kernel = kernel_type(
155
- mp_config,
156
- w_q_param_name="weight_packed",
157
- w_s_param_name="weight_scale",
158
- )
159
-
170
input_quant_config = self._build_input_quant_config()
171
if input_quant_config is not None:
172
layer._humming_input_quant_config = input_quant_config
@@ -167,10 +177,10 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
177
partition_scales = not marlin_repeat_scales_on_all_ranks(
178
self.group_size, row_parallel
179
)
170
- scales_size = input_size // group_size
180
+ scales_and_zp_size = input_size // group_size
181
if partition_scales:
182
assert input_size_per_partition % group_size == 0
173
- scales_size = input_size_per_partition // group_size
183
+ scales_and_zp_size = input_size_per_partition // group_size
184
185
packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
186
layer.register_parameter(
@@ -190,7 +200,7 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
200
)
201
202
scale_data = torch.empty(
193
- output_size_per_partition, scales_size, dtype=params_dtype
203
+ output_size_per_partition, scales_and_zp_size, dtype=params_dtype
204
)
205
if partition_scales:
206
weight_scale = GroupQuantScaleParameter(
@@ -207,6 +217,34 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
217
)
218
layer.register_parameter("weight_scale", weight_scale)
219
220
+ if not self.symmetric:
221
+ packed_output_dim = math.ceil(
222
+ output_size_per_partition * self.num_bits / 32
223
+ )
224
+ zeros_data = torch.zeros(
225
+ packed_output_dim,
226
+ scales_and_zp_size,
227
+ dtype=torch.int32,
228
+ )
229
+ if not partition_scales:
230
+ qzeros = PackedColumnParameter(
231
+ output_dim=0,
232
+ packed_dim=0,
233
+ packed_factor=self.pack_factor,
234
+ weight_loader=weight_loader,
235
+ data=zeros_data,
236
+ )
237
+ else:
238
+ qzeros = PackedvLLMParameter(
239
+ input_dim=1,
240
+ output_dim=0,
241
+ packed_dim=0,
242
+ packed_factor=self.pack_factor,
243
+ weight_loader=weight_loader,
244
+ data=zeros_data,
245
+ )
246
+ layer.register_parameter("weight_zero_point", qzeros)
247
+
248
layer.register_parameter(
249
"weight_shape",
250
BasevLLMParameter(
@@ -215,6 +253,13 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
253
),
254
)
255
256
+ self.kernel = kernel_type(
257
+ mp_config,
258
+ w_q_param_name="weight_packed",
259
+ w_s_param_name="weight_scale",
260
+ w_zp_param_name="weight_zero_point",
261
+ )
262
+
263
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
264
self.kernel.process_weights_after_loading(layer)
265
vllm/model_executor/layers/quantization/utils/humming/linear.py
+13
@@ -101,6 +101,19 @@ def prepare_humming_linear_layer_config(
101
param_dtype=layer.params_dtype,
102
)
103
104
+ if (
105
+ weight_schema.has_zero_point
106
+ and input_schema.a_dtype is not None
107
+ and weight_schema.b_dtype.num_bits == input_schema.a_dtype.num_bits
108
+ ):
109
+ raise ValueError(
110
+ "Humming does not support asymmetric (zero_point) weights "
111
+ "when weight and activation bit widths are equal "
112
+ f"(W{weight_schema.b_dtype.num_bits}"
113
+ f"A{input_schema.a_dtype.num_bits}). "
114
+ "Use symmetric weights or a different activation bit width."
115
+ )
116
+
117
tensors.update(input_tensors)
118
layer.weight_schema = weight_schema
119
input_schema = check_and_fallback_input_schema(
vllm/scalar_type.py
+5
@@ -316,8 +316,13 @@ class ScalarType:
316
317
318
class scalar_types:
319
+ uint2 = ScalarType.uint(2, None)
320
+ uint3 = ScalarType.uint(3, None)
321
int4 = ScalarType.int_(4, None)
322
uint4 = ScalarType.uint(4, None)
323
+ uint5 = ScalarType.uint(5, None)
324
+ uint6 = ScalarType.uint(6, None)
325
+ uint7 = ScalarType.uint(7, None)
326
int8 = ScalarType.int_(8, None)
327
uint8 = ScalarType.uint(8, None)
328
float8_e4m3fn = ScalarType.float_(4, 3, True, NanRepr.EXTD_RANGE_MAX_MIN)