A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 21:48 UTC
[Quantization][Deprecation] Deprecate HQQ (#32681)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
Robert Shaw committed
Jan 21, 2026 at 09:32 UTC
85f55c943cd7b23a613261f35d42c7815976e5a8
8 files changed
-480
csrc/quantization/gptq_marlin/generate_kernels.py
-9
@@ -70,15 +70,6 @@ QUANT_CONFIGS = [
70
"thread_m_blocks": THREAD_M_BLOCKS,
71
"group_blocks": [-1, 2, 4, 8],
72
},
73
- # HQQ
74
- {
75
- "a_type": ["kFloat16"],
76
- "b_type": "kU4",
77
- "thread_configs": THREAD_CONFIGS,
78
- "thread_m_blocks": THREAD_M_BLOCKS,
79
- "group_blocks": [4],
80
- "is_zp_float": True,
81
- },
73
# GPTQ-INT4
74
{
75
"b_type": "kU4B8",
tests/kernels/quantization/test_marlin_gemm.py
-87
@@ -26,7 +26,6 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
26
marlin_make_empty_g_idx,
27
marlin_make_workspace_new,
28
marlin_permute_bias,
29
- marlin_permute_scales,
29
query_marlin_supported_quant_types,
30
)
31
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
@@ -75,8 +74,6 @@ MARLIN_N_CHUNKS = [64, 256]
74
MARLIN_24_K_CHUNKS = [128]
75
MARLIN_24_N_CHUNKS = [512]
76
78
-HQQ_SUPPORTED_GROUP_SIZES = [64]
79
-
77
MARLIN_REPACK_NK_FACTORS = [
78
(4, 8),
79
(7, 5),
@@ -631,90 +628,6 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, quant_type, group_size, mnk_facto
628
assert max_diff < 0.04
629
630
634
-@pytest.mark.skipif(
635
- not is_quant_method_supported("gptq_marlin"),
636
- reason="Marlin is not supported on this GPU type.",
637
-)
638
-@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
639
-@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
640
-@pytest.mark.parametrize("group_size", HQQ_SUPPORTED_GROUP_SIZES)
641
-@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
642
-@pytest.mark.parametrize("use_fp32_reduce", USE_FP32_REDUCE_OPTS)
643
-def test_hqq_marlin_gemm(
644
- k_chunk,
645
- n_chunk,
646
- group_size,
647
- mnk_factors,
648
- use_fp32_reduce,
649
-):
650
- m_factor, n_factor, k_factor = mnk_factors
651
-
652
- size_m = m_factor
653
- size_k = k_chunk * k_factor
654
- size_n = n_chunk * n_factor
655
-
656
- quant_type = scalar_types.uint4
657
-
658
- a_input = rand_data((size_m, size_k))
659
- dev = a_input.device
660
-
661
- b_weight = torch.randint(0, 10, (size_n, size_k), dtype=torch.uint8, device=dev)
662
- scale = rand_data((size_n, size_k // group_size))
663
- zero = rand_data((size_n, size_k // group_size))
664
-
665
- gptq_w_q = gptq_pack(b_weight.transpose(1, 0), 4, size_k, size_n)
666
-
667
- sort_indices = torch.empty(0, dtype=torch.int, device=dev)
668
- marlin_w_q = ops.gptq_marlin_repack(gptq_w_q, sort_indices, size_k, size_n, 4).to(
669
- dev
670
- )
671
- marlin_s = marlin_permute_scales(
672
- scale.transpose(1, 0), size_k, size_n, group_size
673
- ).to(dev)
674
- marlin_zp = marlin_permute_scales(
675
- zero.transpose(1, 0), size_k, size_n, group_size
676
- ).to(dev)
677
-
678
- g_idx = marlin_make_empty_g_idx(dev)
679
- g_idx_sort_indices = marlin_make_empty_g_idx(dev)
680
-
681
- workspace = marlin_make_workspace_new(b_weight.device)
682
-
683
- output = ops.gptq_marlin_gemm(
684
- a_input,
685
- None,
686
- marlin_w_q,
687
- None,
688
- marlin_s,
689
- None,
690
- None,
691
- marlin_zp,
692
- g_idx,
693
- g_idx_sort_indices,
694
- workspace,
695
- quant_type,
696
- a_input.shape[0],
697
- b_weight.shape[0],
698
- a_input.shape[1],
699
- is_k_full=True,
700
- use_fp32_reduce=use_fp32_reduce,
701
- is_zp_float=True,
702
- )
703
-
704
- b_flat = b_weight.reshape(-1, group_size)
705
- zp_flat = zero.reshape(-1, 1)
706
- s_flat = scale.reshape(-1, 1)
707
- dequant = (b_flat - zp_flat) * s_flat
708
-
709
- output_ref = torch.matmul(a_input, dequant.reshape(b_weight.shape).transpose(1, 0))
710
-
711
- torch.cuda.synchronize()
712
-
713
- max_diff = compute_max_diff(output, output_ref)
714
-
715
- assert max_diff < 0.04
716
-
717
-
631
def test_marlin_gemm_subset_input():
632
quant_type = scalar_types.uint4b8
633
group_size = 128
tests/weight_loading/models.txt
-1
@@ -26,5 +26,4 @@ compressed-tensors, nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-W8A8-testing
26
awq, casperhansen/mixtral-instruct-awq, main
27
awq_marlin, casperhansen/mixtral-instruct-awq, main
28
fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
29
-hqq, nm-testing/Llama-3.2-1B-Instruct-HQQ, main
29
None, mgleize/fairseq2-dummy-Llama-3.2-1B, main
\ No newline at end of file
vllm/lora/layers/base_linear.py
-3
@@ -158,9 +158,6 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
158
# marlin
159
elif hasattr(self.base_layer, "B"):
160
return self.base_layer.B
161
- # HQQ marlin
162
- elif hasattr(self.base_layer, "W_q"):
163
- return self.base_layer.W_q
161
else:
162
raise ValueError(f"Unsupported base layer: {self.base_layer}")
163
vllm/lora/layers/utils.py
-3
@@ -41,9 +41,6 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
41
# GPTQ/AWQ
42
elif hasattr(base_layer, "qweight"):
43
return base_layer.qweight.device
44
- # HQQ marlin
45
- elif hasattr(base_layer, "W_q"):
46
- return base_layer.W_q.device
44
# MoE layer
45
elif hasattr(base_layer, "w2_weight"):
46
return base_layer.w2_weight.device
vllm/model_executor/layers/linear.py
-1
@@ -57,7 +57,6 @@ WEIGHT_LOADER_V2_SUPPORTED = [
57
"ModelOptFp8PbWoLinearMethod",
58
"IPEXAWQLinearMethod",
59
"IPEXGPTQLinearMethod",
60
- "HQQMarlinMethod",
60
"QuarkLinearMethod",
61
"ModelOptNvFp4LinearMethod",
62
"PetitNvFp4LinearMethod",
vllm/model_executor/layers/quantization/__init__.py
-4
@@ -26,7 +26,6 @@ QuantizationMethods = Literal[
26
"gptq",
27
"compressed-tensors",
28
"bitsandbytes",
29
- "hqq",
29
"experts_int8",
30
"ipex",
31
"quark",
@@ -48,7 +47,6 @@ DEPRECATED_QUANTIZATION_METHODS = [
47
"bitblas",
48
"gptq_marlin_24",
49
"gptq_bitblas",
51
- "hqq",
50
"experts_int8",
51
"ipex",
52
"rtn",
@@ -133,7 +131,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
131
from .gptq_bitblas import GPTQBitBLASConfig
132
from .gptq_marlin import GPTQMarlinConfig
133
from .gptq_marlin_24 import GPTQMarlin24Config
136
- from .hqq_marlin import HQQMarlinConfig
134
from .inc import INCConfig
135
from .ipex_quant import IPEXConfig
136
from .modelopt import ModelOptFp8Config, ModelOptNvFp4Config
@@ -161,7 +158,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
158
"compressed-tensors": CompressedTensorsConfig,
159
"bitsandbytes": BitsAndBytesConfig,
160
"ptpc_fp8": PTPCFp8Config,
164
- "hqq": HQQMarlinConfig,
161
"experts_int8": ExpertsInt8Config,
162
"ipex": IPEXConfig,
163
"quark": QuarkConfig,
vllm/model_executor/layers/quantization/hqq_marlin.py
deleted
-372
@@ -1,372 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from typing import Any, Optional
5
-
6
-import torch
7
-
8
-from vllm import _custom_ops as ops
9
-from vllm.logger import init_logger
10
-from vllm.model_executor.layers.linear import (
11
- LinearBase,
12
- LinearMethodBase,
13
- UnquantizedLinearMethod,
14
-)
15
-from vllm.model_executor.layers.quantization import QuantizationMethods
16
-from vllm.model_executor.layers.quantization.base_config import (
17
- QuantizationConfig,
18
- QuantizeMethodBase,
19
-)
20
-from vllm.model_executor.layers.quantization.utils.marlin_utils import (
21
- GPTQ_MARLIN_MAX_PARALLEL,
22
- GPTQ_MARLIN_MIN_THREAD_N,
23
- marlin_make_empty_g_idx,
24
- marlin_permute_bias,
25
- marlin_permute_scales,
26
-)
27
-from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
28
- MarlinWorkspace,
29
-)
30
-from vllm.model_executor.layers.quantization.utils.quant_utils import gptq_pack
31
-from vllm.model_executor.parameter import (
32
- BasevLLMParameter,
33
- GroupQuantScaleParameter,
34
- PackedvLLMParameter,
35
-)
36
-from vllm.scalar_type import scalar_types
37
-
38
-logger = init_logger(__name__)
39
-
40
-
41
-class HQQMarlinConfig(QuantizationConfig):
42
- """Config class for HQQ Marlin"""
43
-
44
- def __init__(
45
- self,
46
- weight_bits: int,
47
- group_size: int,
48
- skip_modules: list[str] | None = None,
49
- ) -> None:
50
- super().__init__()
51
- assert group_size == 64, "The only supported HQQ group size is currently 64."
52
- assert weight_bits == 4, (
53
- "The only supported HQQ quantization bitsize is currently 4."
54
- )
55
-
56
- self.weight_bits = weight_bits
57
- self.group_size = group_size
58
- self.pack_factor = 32 // weight_bits # packed into int32 in GPTQ format
59
- self.quant_type = scalar_types.uint4
60
- self.skip_modules = skip_modules
61
-
62
- def __repr__(self) -> str:
63
- return (
64
- f"HQQMarlinConfig(quant_type={self.quant_type}, "
65
- f"group_size={self.group_size})"
66
- )
67
-
68
- @classmethod
69
- def get_name(cls) -> QuantizationMethods:
70
- return "hqq"
71
-
72
- @classmethod
73
- def get_supported_act_dtypes(cls) -> list[torch.dtype]:
74
- return [torch.half, torch.bfloat16]
75
-
76
- @classmethod
77
- def get_min_capability(cls) -> int:
78
- return 80
79
-
80
- @classmethod
81
- def get_config_filenames(cls) -> list[str]:
82
- return ["quantize_config.json"]
83
-
84
- @classmethod
85
- def from_config(cls, config: dict[str, Any]) -> "HQQMarlinConfig":
86
- wq_params = config["quant_config"]["weight_quant_params"]
87
- weight_bits = cls.get_from_keys(wq_params, ["nbits"])
88
- group_size = cls.get_from_keys(wq_params, ["group_size"])
89
- skip_modules = config["skip_modules"]
90
- return cls(weight_bits, group_size, skip_modules)
91
-
92
- def is_layer_skipped(self, prefix: str) -> bool:
93
- # Split the prefix into its dot-separated components
94
- components = prefix.split(".")
95
-
96
- # Check if any of the skip modules exactly matches any component
97
- return self.skip_modules is not None and any(
98
- module_name in components for module_name in self.skip_modules
99
- )
100
-
101
- def get_quant_method(
102
- self, layer: torch.nn.Module, prefix: str
103
- ) -> Optional["QuantizeMethodBase"]:
104
- if isinstance(layer, LinearBase):
105
- if self.is_layer_skipped(prefix):
106
- return UnquantizedLinearMethod()
107
- return HQQMarlinMethod(self)
108
- return None
109
-
110
-
111
-# Empty HQQ parameter, will be ignored during loading
112
-class HQQEmptyParameter(BasevLLMParameter):
113
- def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs):
114
- pass
115
-
116
- def load_row_parallel_weight(self, loaded_weight: torch.Tensor):
117
- pass
118
-
119
- def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs):
120
- pass
121
-
122
-
123
-def error_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
124
- raise ValueError("No loader provided for HQQ parameter!")
125
-
126
-
127
-# HQQ packing creates issues with sharding - therefore, prior to loading, we
128
-# repack to GPTQ. We also reshape the weights to their proper GPTQ shape.
129
-class HQQweightParameter(PackedvLLMParameter):
130
- # unpack function from https://github.com/mobiusml/hqq
131
- def unpack_4bit_u8(self, W_q: torch.Tensor) -> torch.Tensor: # uint8/2 > uint8
132
- assert self.weight_bits == 4, "Unsupported quant bitsize (must be 4)"
133
-
134
- dtype = torch.uint8
135
- step = W_q.shape[0]
136
- tmp = torch.empty([2 * step, W_q.shape[1]], dtype=dtype, device=W_q.device)
137
- tmp[:step] = (W_q & 0b11110000) >> 4
138
- tmp[step:] = W_q & 0b00001111
139
- return tmp
140
-
141
- def __init__(self, packed_factor: int, packed_dim: int, weight_bits: int, **kwargs):
142
- super().__init__(packed_factor, packed_dim, None, **kwargs)
143
- self.weight_bits = weight_bits
144
- self.input_shape = self.shape[self.input_dim] * self.packed_factor
145
- self.output_shape = self.shape[self.output_dim]
146
-
147
- def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs):
148
- loaded_weight = self.unpack_4bit_u8(loaded_weight)
149
- loaded_weight = loaded_weight.reshape(-1, self.input_shape).transpose(1, 0)
150
- loaded_weight = gptq_pack(
151
- loaded_weight,
152
- self.weight_bits,
153
- loaded_weight.shape[0],
154
- loaded_weight.shape[1],
155
- )
156
- super().load_merged_column_weight(loaded_weight, **kwargs)
157
-
158
- def load_row_parallel_weight(self, loaded_weight: torch.Tensor):
159
- loaded_weight = self.unpack_4bit_u8(loaded_weight)
160
- loaded_weight = loaded_weight.reshape(self.output_shape, -1).transpose(1, 0)
161
- loaded_weight = gptq_pack(
162
- loaded_weight,
163
- self.weight_bits,
164
- loaded_weight.shape[0],
165
- loaded_weight.shape[1],
166
- )
167
- super().load_row_parallel_weight(loaded_weight)
168
-
169
- def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs):
170
- loaded_weight = self.unpack_4bit_u8(loaded_weight)
171
- loaded_weight = loaded_weight.reshape(-1, self.input_shape).transpose(1, 0)
172
- loaded_weight = gptq_pack(
173
- loaded_weight,
174
- self.weight_bits,
175
- loaded_weight.shape[0],
176
- loaded_weight.shape[1],
177
- )
178
- super().load_qkv_weight(loaded_weight, **kwargs)
179
-
180
-
181
-# Zero points and scales in HQQ must also be reshaped to correspond to W_q's
182
-# GPTQ shape (transposed - we transpose them too when processing weights).
183
-class HQQZeroScaleParameter(GroupQuantScaleParameter):
184
- def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs):
185
- loaded_weight = loaded_weight.reshape(-1, self.shape[1])
186
- super().load_merged_column_weight(loaded_weight, **kwargs)
187
-
188
- def load_row_parallel_weight(self, loaded_weight: torch.Tensor):
189
- loaded_weight = loaded_weight.reshape(self.shape[0], -1)
190
- super().load_row_parallel_weight(loaded_weight)
191
-
192
- def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs):
193
- loaded_weight = loaded_weight.reshape(-1, self.shape[1])
194
- super().load_qkv_weight(loaded_weight, **kwargs)
195
-
196
-
197
-class HQQMarlinMethod(LinearMethodBase):
198
- """Linear method for HQQ Marlin."""
199
-
200
- def __init__(
201
- self,
202
- quant_config: HQQMarlinConfig,
203
- ):
204
- self.quant_config = quant_config
205
-
206
- def create_weights(
207
- self,
208
- layer: torch.nn.Module,
209
- input_size_per_partition: int,
210
- output_partition_sizes: list[int],
211
- input_size: int,
212
- output_size: int,
213
- params_dtype: torch.dtype,
214
- **extra_weight_attrs,
215
- ) -> None:
216
- self.output_size_per_partition = sum(output_partition_sizes)
217
- self.input_size_per_partition = input_size_per_partition
218
-
219
- weight_loader = extra_weight_attrs.get("weight_loader", error_loader)
220
-
221
- self.scales_and_zp_size = (
222
- input_size_per_partition // self.quant_config.group_size
223
- )
224
-
225
- qweight = HQQweightParameter(
226
- data=torch.empty(
227
- self.input_size_per_partition // self.quant_config.pack_factor,
228
- self.output_size_per_partition,
229
- dtype=torch.int32,
230
- ),
231
- input_dim=0,
232
- output_dim=1,
233
- packed_dim=0,
234
- packed_factor=self.quant_config.pack_factor,
235
- weight_bits=self.quant_config.weight_bits,
236
- weight_loader=weight_loader,
237
- )
238
-
239
- zeros = HQQZeroScaleParameter(
240
- data=torch.empty(
241
- self.output_size_per_partition,
242
- self.scales_and_zp_size,
243
- dtype=params_dtype,
244
- ),
245
- input_dim=1,
246
- output_dim=0,
247
- weight_loader=weight_loader,
248
- )
249
-
250
- scales = HQQZeroScaleParameter(
251
- data=torch.empty(
252
- self.output_size_per_partition,
253
- self.scales_and_zp_size,
254
- dtype=params_dtype,
255
- ),
256
- input_dim=1,
257
- output_dim=0,
258
- weight_loader=weight_loader,
259
- )
260
-
261
- layer.register_parameter("W_q", qweight)
262
- layer.register_parameter("zero", zeros)
263
- layer.register_parameter("scale", scales)
264
-
265
- # Ignore extra parameters in the HQQ model.
266
- # To be added as needed.
267
- ignore_parameters = (
268
- "axis",
269
- "channel_wise",
270
- "compute_dtype",
271
- "encoded_state_dict",
272
- "group_size",
273
- "nbits",
274
- "offload_meta",
275
- "optimize",
276
- "packing",
277
- "quant_scale",
278
- "quant_zero",
279
- "round_zero",
280
- "shape",
281
- "stores_quant_config",
282
- "unpack_view_dtype",
283
- "view_as_float",
284
- )
285
- for name in ignore_parameters:
286
- layer.register_parameter(
287
- name,
288
- HQQEmptyParameter(data=torch.empty(0), weight_loader=weight_loader),
289
- )
290
-
291
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
292
- dev = layer.W_q.device
293
-
294
- # Repack to Marlin
295
- sort_indices = torch.empty(0, dtype=torch.int, device=dev)
296
- marlin_w_q = ops.gptq_marlin_repack(
297
- layer.W_q,
298
- sort_indices,
299
- self.input_size_per_partition,
300
- self.output_size_per_partition,
301
- self.quant_config.weight_bits,
302
- ).to(dev)
303
- marlin_s = marlin_permute_scales(
304
- layer.scale.transpose(1, 0),
305
- self.input_size_per_partition,
306
- self.output_size_per_partition,
307
- self.quant_config.group_size,
308
- ).to(dev)
309
- marlin_zp = marlin_permute_scales(
310
- layer.zero.transpose(1, 0),
311
- self.input_size_per_partition,
312
- self.output_size_per_partition,
313
- self.quant_config.group_size,
314
- ).to(dev)
315
-
316
- layer.g_idx = marlin_make_empty_g_idx(dev)
317
- layer.g_idx_sort_indices = marlin_make_empty_g_idx(dev)
318
-
319
- layer.marlin_qweight = marlin_w_q
320
- layer.marlin_zeros = marlin_zp
321
- layer.marlin_scales = marlin_s
322
-
323
- if hasattr(layer, "bias") and layer.bias is not None:
324
- layer.bias.data = marlin_permute_bias(layer.bias)
325
-
326
- def apply(
327
- self,
328
- layer: torch.nn.Module,
329
- x: torch.Tensor,
330
- bias: torch.Tensor | None = None,
331
- ) -> torch.Tensor:
332
- workspace = MarlinWorkspace(
333
- self.output_size_per_partition,
334
- GPTQ_MARLIN_MIN_THREAD_N,
335
- GPTQ_MARLIN_MAX_PARALLEL,
336
- )
337
-
338
- scales = layer.marlin_scales
339
- zeros = layer.marlin_zeros
340
- orig_type = x.dtype
341
-
342
- if orig_type != torch.float16:
343
- x = x.to(torch.float16)
344
- scales = scales.to(torch.float16)
345
- zeros = zeros.to(torch.float16)
346
-
347
- marlin_out = ops.gptq_marlin_gemm(
348
- x,
349
- None,
350
- layer.marlin_qweight,
351
- bias,
352
- scales,
353
- None,
354
- None,
355
- zeros,
356
- layer.g_idx,
357
- layer.g_idx_sort_indices,
358
- workspace.scratch,
359
- scalar_types.uint4,
360
- x.shape[0],
361
- self.output_size_per_partition,
362
- self.input_size_per_partition,
363
- True, # is_k_full
364
- False, # use atomic add
365
- True, # use 32-bit reduce
366
- True, # use float zp
367
- )
368
-
369
- if orig_type != torch.float16:
370
- marlin_out = marlin_out.to(orig_type)
371
-
372
- return marlin_out