[Quant] Enable humming w[2-7]a[4,8] inference with compressed-tensors (#46390)
Signed-off-by: HDCharles <charlesdavidhernandez@gmail.com>
HDCharles committed
Jul 14, 2026 at 22:22 UTC
9b2be4e9a55033a410a6f9e7c18eda5ce9105162
8 files changed
+544
-3
tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM"
2
+accuracy_threshold: 0.85
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: >-
6
+ --enforce-eager
7
+ --max-model-len 4096
8
+ --linear-backend humming
tests/evals/gsm8k/configs/humming/config-act-int8.txt
+1
@@ -2,3 +2,4 @@ Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
2
Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
3
Qwen3-30B-A3B-AWQ-humming.yaml
4
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
5
+Qwen3-4B-mixed-quant-RTN-humming.yaml
vllm/model_executor/kernels/linear/mixed_precision/humming.py
+4
-1
@@ -45,7 +45,10 @@ class HummingLinearKernel(MPLinearKernel):
45
quant_config["has_zero_point"] = True
46
47
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
48
- prepare_humming_layer(layer, quant_config)
48
+ input_quant_config = getattr(layer, "_humming_input_quant_config", None)
49
+ prepare_humming_layer(
50
+ layer, quant_config, input_quant_config=input_quant_config
51
+ )
52
53
def apply_weights(
54
self,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+62
@@ -47,6 +47,8 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
47
CompressedTensorsW8A8Int8,
48
CompressedTensorsW8A8Mxfp8,
49
CompressedTensorsW8A16Fp8,
50
+ CompressedTensorsWNA4Int,
51
+ CompressedTensorsWNA8Int,
52
CompressedTensorsWNA8O8Int,
53
CompressedTensorsWNA16,
54
)
@@ -682,6 +684,41 @@ class CompressedTensorsConfig(QuantizationConfig):
684
)
685
return is_intN_weight and (is_static_int8_in and is_static_int8_out)
686
687
+ @staticmethod
688
+ def _is_wNaM_int(
689
+ weight_quant: QuantizationArgs,
690
+ input_quant: QuantizationArgs | None,
691
+ format: str | None,
692
+ ) -> bool:
693
+ """Weight N-bit INT with symmetric dynamic INT activation quant
694
+ via Humming kernel."""
695
+ if input_quant is None:
696
+ return False
697
+ is_pack_format = format == CompressionFormat.pack_quantized.value
698
+ is_channel_group = weight_quant.strategy in (
699
+ QuantizationStrategy.CHANNEL.value,
700
+ QuantizationStrategy.GROUP.value,
701
+ )
702
+ is_int_N_weight = (
703
+ weight_quant.type == QuantizationType.INT and not weight_quant.dynamic
704
+ )
705
+ is_int_input = input_quant.type == QuantizationType.INT
706
+ is_symmetric_input = input_quant.symmetric
707
+ is_dynamic_input = input_quant.dynamic
708
+ is_per_token_or_group_input = input_quant.strategy in (
709
+ QuantizationStrategy.TOKEN.value,
710
+ QuantizationStrategy.GROUP.value,
711
+ )
712
+ return (
713
+ is_int_N_weight
714
+ and is_channel_group
715
+ and is_pack_format
716
+ and is_int_input
717
+ and is_symmetric_input
718
+ and is_dynamic_input
719
+ and is_per_token_or_group_input
720
+ )
721
+
722
def _get_scheme_from_parts(
723
self,
724
weight_quant: QuantizationArgs,
@@ -733,6 +770,31 @@ class CompressedTensorsConfig(QuantizationConfig):
770
quant_format=format,
771
)
772
773
+ if (
774
+ self._is_wNaM_int(weight_quant, input_quant, format)
775
+ and input_quant.num_bits == 8
776
+ ):
777
+ return CompressedTensorsWNA8Int(
778
+ num_bits=weight_quant.num_bits,
779
+ strategy=weight_quant.strategy,
780
+ group_size=weight_quant.group_size,
781
+ input_quant=input_quant,
782
+ layer_name=layer_name,
783
+ quant_format=format,
784
+ )
785
+ if (
786
+ self._is_wNaM_int(weight_quant, input_quant, format)
787
+ and input_quant.num_bits == 4
788
+ ):
789
+ return CompressedTensorsWNA4Int(
790
+ num_bits=weight_quant.num_bits,
791
+ strategy=weight_quant.strategy,
792
+ group_size=weight_quant.group_size,
793
+ input_quant=input_quant,
794
+ layer_name=layer_name,
795
+ quant_format=format,
796
+ )
797
+
798
if self._is_wNa16_group_channel(weight_quant, input_quant) and (
799
format == CompressionFormat.pack_quantized.value
800
):
vllm/model_executor/layers/quantization/compressed_tensors/schemes/__init__.py
+4
@@ -10,6 +10,8 @@ from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
10
from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
11
from .compressed_tensors_w8a8_mxfp8 import CompressedTensorsW8A8Mxfp8
12
from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8
13
+from .compressed_tensors_wNa4 import CompressedTensorsWNA4Int
14
+from .compressed_tensors_wNa8 import CompressedTensorsWNA8Int
15
from .compressed_tensors_wNa8o8 import CompressedTensorsWNA8O8Int
16
from .compressed_tensors_wNa16 import CompressedTensorsWNA16
17
@@ -25,4 +27,6 @@ __all__ = [
27
"CompressedTensorsW4A8Int",
28
"CompressedTensorsW4A8Fp8",
29
"CompressedTensorsW8A8Mxfp8",
30
+ "CompressedTensorsWNA4Int",
31
+ "CompressedTensorsWNA8Int",
32
]
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa4.py
new
+228
@@ -0,0 +1,228 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Weight N-bit INT scheme with symmetric INT4 activation quant via Humming.
4
+
5
+Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit)
6
+with INT4 symmetric dynamic per-token/per-group input activation
7
+quantization. Static, per-tensor, and asymmetric activation quantization
8
+are not supported.
9
+"""
10
+
11
+import math
12
+from collections.abc import Callable
13
+from fractions import Fraction
14
+
15
+import torch
16
+from compressed_tensors.quantization import (
17
+ QuantizationArgs,
18
+ QuantizationStrategy,
19
+)
20
+
21
+from vllm.logger import init_logger
22
+from vllm.model_executor.kernels.linear import (
23
+ MPLinearLayerConfig,
24
+ choose_mp_linear_kernel,
25
+)
26
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
27
+ CompressedTensorsScheme,
28
+)
29
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
30
+ WNA16_SUPPORTED_TYPES_MAP,
31
+)
32
+from vllm.model_executor.layers.quantization.utils.marlin_utils import (
33
+ marlin_repeat_scales_on_all_ranks,
34
+)
35
+from vllm.model_executor.parameter import (
36
+ BasevLLMParameter,
37
+ ChannelQuantScaleParameter,
38
+ GroupQuantScaleParameter,
39
+ PackedvLLMParameter,
40
+)
41
+
42
+logger = init_logger(__name__)
43
+
44
+__all__ = ["CompressedTensorsWNA4Int"]
45
+
46
+
47
+class CompressedTensorsWNA4Int(CompressedTensorsScheme):
48
+ _kernel_backends_being_used: set[str] = set()
49
+
50
+ def __init__(
51
+ self,
52
+ num_bits: int,
53
+ strategy: str,
54
+ group_size: int | None = None,
55
+ input_quant: QuantizationArgs | None = None,
56
+ layer_name: str | None = None,
57
+ quant_format: str = "pack-quantized",
58
+ ):
59
+ self.num_bits = num_bits
60
+ self.pack_factor = Fraction(32, num_bits)
61
+ self.strategy = strategy
62
+ self.group_size = -1 if group_size is None else group_size
63
+ self.input_quant = input_quant
64
+ self.layer_name = layer_name
65
+ self.quant_format = quant_format
66
+
67
+ if num_bits not in WNA16_SUPPORTED_TYPES_MAP:
68
+ raise ValueError(
69
+ f"Unsupported num_bits = {num_bits} for WNA4Int; "
70
+ f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}"
71
+ )
72
+ self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
73
+
74
+ if input_quant is not None:
75
+ if not input_quant.symmetric:
76
+ raise ValueError(
77
+ "WNA4Int requires symmetric activation quantization, "
78
+ f"got symmetric={input_quant.symmetric}"
79
+ )
80
+ if not input_quant.dynamic:
81
+ raise ValueError(
82
+ "WNA4Int requires dynamic activation quantization, "
83
+ f"got dynamic={input_quant.dynamic}"
84
+ )
85
+ if input_quant.strategy not in (
86
+ QuantizationStrategy.TOKEN.value,
87
+ QuantizationStrategy.GROUP.value,
88
+ ):
89
+ raise ValueError(
90
+ "WNA4Int requires per-token or per-group activation "
91
+ f"quantization, got strategy={input_quant.strategy}"
92
+ )
93
+
94
+ @classmethod
95
+ def get_min_capability(cls) -> int:
96
+ return 75
97
+
98
+ def _build_input_quant_config(self) -> dict | None:
99
+ """Build the config dict that HummingInputSchema.from_config expects."""
100
+ if self.input_quant is None:
101
+ return None
102
+ iq = self.input_quant
103
+ type_val = iq.type.value if hasattr(iq.type, "value") else iq.type
104
+ strategy_val = (
105
+ iq.strategy.value if hasattr(iq.strategy, "value") else iq.strategy
106
+ )
107
+ return {
108
+ "num_bits": iq.num_bits,
109
+ "type": type_val,
110
+ "strategy": strategy_val,
111
+ "symmetric": iq.symmetric,
112
+ "dynamic": iq.dynamic,
113
+ "group_size": iq.group_size or 0,
114
+ "quant_method": "compressed-tensors",
115
+ "format": self.quant_format,
116
+ }
117
+
118
+ def create_weights(
119
+ self,
120
+ layer: torch.nn.Module,
121
+ output_size: int,
122
+ input_size: int,
123
+ output_partition_sizes: list[int],
124
+ input_size_per_partition: int,
125
+ params_dtype: torch.dtype,
126
+ weight_loader: Callable,
127
+ **kwargs,
128
+ ):
129
+ output_size_per_partition = sum(output_partition_sizes)
130
+ layer.input_size_per_partition = input_size_per_partition
131
+ layer.output_size_per_partition = output_size_per_partition
132
+ layer.output_partition_sizes = output_partition_sizes
133
+ layer.params_dtype = params_dtype
134
+ if not hasattr(layer, "has_bias"):
135
+ layer.has_bias = False
136
+
137
+ mp_config = MPLinearLayerConfig(
138
+ full_weight_shape=(input_size, output_size),
139
+ partition_weight_shape=(
140
+ input_size_per_partition,
141
+ output_size_per_partition,
142
+ ),
143
+ weight_type=self.quant_type,
144
+ act_type=params_dtype,
145
+ group_size=self.group_size,
146
+ zero_points=False,
147
+ has_g_idx=False,
148
+ )
149
+
150
+ kernel_type = choose_mp_linear_kernel(mp_config)
151
+ if kernel_type.__name__ not in self._kernel_backends_being_used:
152
+ logger.info("Using %s for CompressedTensorsWNA4Int", kernel_type.__name__)
153
+ self._kernel_backends_being_used.add(kernel_type.__name__)
154
+
155
+ self.kernel = kernel_type(
156
+ mp_config,
157
+ w_q_param_name="weight_packed",
158
+ w_s_param_name="weight_scale",
159
+ )
160
+
161
+ input_quant_config = self._build_input_quant_config()
162
+ if input_quant_config is not None:
163
+ layer._humming_input_quant_config = input_quant_config
164
+
165
+ # --- weight parameters ---
166
+ group_size = self.group_size if self.group_size != -1 else input_size
167
+ row_parallel = input_size != input_size_per_partition
168
+ partition_scales = not marlin_repeat_scales_on_all_ranks(
169
+ False, self.group_size, row_parallel
170
+ )
171
+ scales_size = input_size // group_size
172
+ if partition_scales:
173
+ assert input_size_per_partition % group_size == 0
174
+ scales_size = input_size_per_partition // group_size
175
+
176
+ packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
177
+ layer.register_parameter(
178
+ "weight_packed",
179
+ PackedvLLMParameter(
180
+ input_dim=1,
181
+ output_dim=0,
182
+ packed_factor=self.pack_factor,
183
+ packed_dim=1,
184
+ weight_loader=weight_loader,
185
+ data=torch.empty(
186
+ output_size_per_partition,
187
+ packed_input_dim,
188
+ dtype=torch.int32,
189
+ ),
190
+ ),
191
+ )
192
+
193
+ scale_data = torch.empty(
194
+ output_size_per_partition, scales_size, dtype=params_dtype
195
+ )
196
+ if partition_scales:
197
+ weight_scale = GroupQuantScaleParameter(
198
+ data=scale_data,
199
+ output_dim=0,
200
+ input_dim=1,
201
+ weight_loader=weight_loader,
202
+ )
203
+ else:
204
+ weight_scale = ChannelQuantScaleParameter(
205
+ data=scale_data,
206
+ output_dim=0,
207
+ weight_loader=weight_loader,
208
+ )
209
+ layer.register_parameter("weight_scale", weight_scale)
210
+
211
+ layer.register_parameter(
212
+ "weight_shape",
213
+ BasevLLMParameter(
214
+ data=torch.empty(2, dtype=torch.int64),
215
+ weight_loader=weight_loader,
216
+ ),
217
+ )
218
+
219
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
220
+ self.kernel.process_weights_after_loading(layer)
221
+
222
+ def apply_weights(
223
+ self,
224
+ layer: torch.nn.Module,
225
+ x: torch.Tensor,
226
+ bias: torch.Tensor | None,
227
+ ) -> torch.Tensor:
228
+ return self.kernel.apply_weights(layer, x, bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa8.py
new
+228
@@ -0,0 +1,228 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Weight N-bit INT scheme with symmetric INT8 activation quant via Humming.
4
+
5
+Handles compressed-tensors pack-quantized INT weight checkpoints (2-8 bit)
6
+with INT8 symmetric dynamic per-token/per-group input activation
7
+quantization. Static, per-tensor, and asymmetric activation quantization
8
+are not supported.
9
+"""
10
+
11
+import math
12
+from collections.abc import Callable
13
+from fractions import Fraction
14
+
15
+import torch
16
+from compressed_tensors.quantization import (
17
+ QuantizationArgs,
18
+ QuantizationStrategy,
19
+)
20
+
21
+from vllm.logger import init_logger
22
+from vllm.model_executor.kernels.linear import (
23
+ MPLinearLayerConfig,
24
+ choose_mp_linear_kernel,
25
+)
26
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
27
+ CompressedTensorsScheme,
28
+)
29
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
30
+ WNA16_SUPPORTED_TYPES_MAP,
31
+)
32
+from vllm.model_executor.layers.quantization.utils.marlin_utils import (
33
+ marlin_repeat_scales_on_all_ranks,
34
+)
35
+from vllm.model_executor.parameter import (
36
+ BasevLLMParameter,
37
+ ChannelQuantScaleParameter,
38
+ GroupQuantScaleParameter,
39
+ PackedvLLMParameter,
40
+)
41
+
42
+logger = init_logger(__name__)
43
+
44
+__all__ = ["CompressedTensorsWNA8Int"]
45
+
46
+
47
+class CompressedTensorsWNA8Int(CompressedTensorsScheme):
48
+ _kernel_backends_being_used: set[str] = set()
49
+
50
+ def __init__(
51
+ self,
52
+ num_bits: int,
53
+ strategy: str,
54
+ group_size: int | None = None,
55
+ input_quant: QuantizationArgs | None = None,
56
+ layer_name: str | None = None,
57
+ quant_format: str = "pack-quantized",
58
+ ):
59
+ self.num_bits = num_bits
60
+ self.pack_factor = Fraction(32, num_bits)
61
+ self.strategy = strategy
62
+ self.group_size = -1 if group_size is None else group_size
63
+ self.input_quant = input_quant
64
+ self.layer_name = layer_name
65
+ self.quant_format = quant_format
66
+
67
+ if num_bits not in WNA16_SUPPORTED_TYPES_MAP:
68
+ raise ValueError(
69
+ f"Unsupported num_bits = {num_bits} for WNA8Int; "
70
+ f"supported = {sorted(WNA16_SUPPORTED_TYPES_MAP)}"
71
+ )
72
+ self.quant_type = WNA16_SUPPORTED_TYPES_MAP[num_bits]
73
+
74
+ if input_quant is not None:
75
+ if not input_quant.symmetric:
76
+ raise ValueError(
77
+ "WNA8Int requires symmetric activation quantization, "
78
+ f"got symmetric={input_quant.symmetric}"
79
+ )
80
+ if not input_quant.dynamic:
81
+ raise ValueError(
82
+ "WNA8Int requires dynamic activation quantization, "
83
+ f"got dynamic={input_quant.dynamic}"
84
+ )
85
+ if input_quant.strategy not in (
86
+ QuantizationStrategy.TOKEN.value,
87
+ QuantizationStrategy.GROUP.value,
88
+ ):
89
+ raise ValueError(
90
+ "WNA8Int requires per-token or per-group activation "
91
+ f"quantization, got strategy={input_quant.strategy}"
92
+ )
93
+
94
+ @classmethod
95
+ def get_min_capability(cls) -> int:
96
+ return 75
97
+
98
+ def _build_input_quant_config(self) -> dict | None:
99
+ """Build the config dict that HummingInputSchema.from_config expects."""
100
+ if self.input_quant is None:
101
+ return None
102
+ iq = self.input_quant
103
+ type_val = iq.type.value if hasattr(iq.type, "value") else iq.type
104
+ strategy_val = (
105
+ iq.strategy.value if hasattr(iq.strategy, "value") else iq.strategy
106
+ )
107
+ return {
108
+ "num_bits": iq.num_bits,
109
+ "type": type_val,
110
+ "strategy": strategy_val,
111
+ "symmetric": iq.symmetric,
112
+ "dynamic": iq.dynamic,
113
+ "group_size": iq.group_size or 0,
114
+ "quant_method": "compressed-tensors",
115
+ "format": self.quant_format,
116
+ }
117
+
118
+ def create_weights(
119
+ self,
120
+ layer: torch.nn.Module,
121
+ output_size: int,
122
+ input_size: int,
123
+ output_partition_sizes: list[int],
124
+ input_size_per_partition: int,
125
+ params_dtype: torch.dtype,
126
+ weight_loader: Callable,
127
+ **kwargs,
128
+ ):
129
+ output_size_per_partition = sum(output_partition_sizes)
130
+ layer.input_size_per_partition = input_size_per_partition
131
+ layer.output_size_per_partition = output_size_per_partition
132
+ layer.output_partition_sizes = output_partition_sizes
133
+ layer.params_dtype = params_dtype
134
+ if not hasattr(layer, "has_bias"):
135
+ layer.has_bias = False
136
+
137
+ mp_config = MPLinearLayerConfig(
138
+ full_weight_shape=(input_size, output_size),
139
+ partition_weight_shape=(
140
+ input_size_per_partition,
141
+ output_size_per_partition,
142
+ ),
143
+ weight_type=self.quant_type,
144
+ act_type=params_dtype,
145
+ group_size=self.group_size,
146
+ zero_points=False,
147
+ has_g_idx=False,
148
+ )
149
+
150
+ kernel_type = choose_mp_linear_kernel(mp_config)
151
+ if kernel_type.__name__ not in self._kernel_backends_being_used:
152
+ logger.info("Using %s for CompressedTensorsWNA8Int", kernel_type.__name__)
153
+ self._kernel_backends_being_used.add(kernel_type.__name__)
154
+
155
+ self.kernel = kernel_type(
156
+ mp_config,
157
+ w_q_param_name="weight_packed",
158
+ w_s_param_name="weight_scale",
159
+ )
160
+
161
+ input_quant_config = self._build_input_quant_config()
162
+ if input_quant_config is not None:
163
+ layer._humming_input_quant_config = input_quant_config
164
+
165
+ # --- weight parameters ---
166
+ group_size = self.group_size if self.group_size != -1 else input_size
167
+ row_parallel = input_size != input_size_per_partition
168
+ partition_scales = not marlin_repeat_scales_on_all_ranks(
169
+ False, self.group_size, row_parallel
170
+ )
171
+ scales_size = input_size // group_size
172
+ if partition_scales:
173
+ assert input_size_per_partition % group_size == 0
174
+ scales_size = input_size_per_partition // group_size
175
+
176
+ packed_input_dim = math.ceil(input_size_per_partition * self.num_bits / 32)
177
+ layer.register_parameter(
178
+ "weight_packed",
179
+ PackedvLLMParameter(
180
+ input_dim=1,
181
+ output_dim=0,
182
+ packed_factor=self.pack_factor,
183
+ packed_dim=1,
184
+ weight_loader=weight_loader,
185
+ data=torch.empty(
186
+ output_size_per_partition,
187
+ packed_input_dim,
188
+ dtype=torch.int32,
189
+ ),
190
+ ),
191
+ )
192
+
193
+ scale_data = torch.empty(
194
+ output_size_per_partition, scales_size, dtype=params_dtype
195
+ )
196
+ if partition_scales:
197
+ weight_scale = GroupQuantScaleParameter(
198
+ data=scale_data,
199
+ output_dim=0,
200
+ input_dim=1,
201
+ weight_loader=weight_loader,
202
+ )
203
+ else:
204
+ weight_scale = ChannelQuantScaleParameter(
205
+ data=scale_data,
206
+ output_dim=0,
207
+ weight_loader=weight_loader,
208
+ )
209
+ layer.register_parameter("weight_scale", weight_scale)
210
+
211
+ layer.register_parameter(
212
+ "weight_shape",
213
+ BasevLLMParameter(
214
+ data=torch.empty(2, dtype=torch.int64),
215
+ weight_loader=weight_loader,
216
+ ),
217
+ )
218
+
219
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
220
+ self.kernel.process_weights_after_loading(layer)
221
+
222
+ def apply_weights(
223
+ self,
224
+ layer: torch.nn.Module,
225
+ x: torch.Tensor,
226
+ bias: torch.Tensor | None,
227
+ ) -> torch.Tensor:
228
+ return self.kernel.apply_weights(layer, x, bias)
vllm/model_executor/layers/quantization/utils/humming_utils.py
+9
-2
@@ -435,7 +435,11 @@ def convert_linear_layer_to_humming_standard(
435
setattr(layer, name, param)
436
437
438
-def prepare_humming_layer(layer: LinearBase, quant_config: dict):
438
+def prepare_humming_layer(
439
+ layer: LinearBase,
440
+ quant_config: dict,
441
+ input_quant_config: dict | None = None,
442
+):
443
from vllm.utils.humming import (
444
BaseWeightSchema,
445
HummingInputSchema,
@@ -443,7 +447,10 @@ def prepare_humming_layer(layer: LinearBase, quant_config: dict):
447
)
448
449
weight_schema = BaseWeightSchema.from_config(quant_config)
446
- input_schema = HummingInputSchema()
450
+ if input_quant_config is not None:
451
+ input_schema = HummingInputSchema.from_config(input_quant_config)
452
+ else:
453
+ input_schema = HummingInputSchema()
454
455
# ReplicatedLinear has no TP partitioning and so does not set
456
# input_size_per_partition; for it that is just input_size. Use hasattr