@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 247d1a32ea
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 11:15 UTC

[Quantization][Deprecation] Remove BitBlas (#32683)

Signed-off-by: Robert Shaw <robshaw@redhat.com> Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>

Robert Shaw committed Jan 28, 2026 at 03:06 UTC 247d1a32ea63f42196e1023ea01362383dc6325e
15 files changed +2 -2030
benchmarks/kernels/benchmark_bitblas.py deleted
-244
@@ -1,244 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -# Copyright (c) Microsoft Corporation.
4 -# Licensed under the MIT License.
5 -
6 -from packaging import version
7 -
8 -from vllm.model_executor.layers.quantization.utils.bitblas_utils import (
9 - MINIMUM_BITBLAS_VERSION,
10 -)
11 -
12 -try:
13 - import bitblas
14 -
15 - if version.parse(bitblas.__version__) < version.parse(MINIMUM_BITBLAS_VERSION):
16 - raise ImportError(
17 - "bitblas version is wrong. Please "
18 - f"install bitblas>={MINIMUM_BITBLAS_VERSION}"
19 - )
20 -except ImportError as e:
21 - bitblas_import_exception = e
22 - raise ValueError(
23 - "Trying to use the bitblas backend, but could not import"
24 - f"with the following error: {bitblas_import_exception}. "
25 - "Please install bitblas through the following command: "
26 - f"`pip install bitblas>={MINIMUM_BITBLAS_VERSION}`"
27 - ) from bitblas_import_exception
28 -
29 -from bitblas import Matmul, MatmulConfig, auto_detect_nvidia_target
30 -
31 -from vllm.utils.argparse_utils import FlexibleArgumentParser
32 -
33 -parser = FlexibleArgumentParser(
34 - description="Benchmark BitBLAS int4 on a specific target."
35 -)
36 -
37 -# Add arguments to the parser
38 -parser.add_argument(
39 - "--target",
40 - type=str,
41 - default=auto_detect_nvidia_target(),
42 - help="Specify the target device for benchmarking.",
43 -)
44 -parser.add_argument(
45 - "--group_size", type=int, default=None, help="Group size for grouped quantization."
46 -)
47 -parser.add_argument(
48 - "--A_dtype",
49 - type=str,
50 - default="float16",
51 - choices=["float16", "float32", "float64", "int32", "int8"],
52 - help="Data type of activation A.",
53 -)
54 -parser.add_argument(
55 - "--W_dtype",
56 - type=str,
57 - default="int4",
58 - choices=[
59 - "float16",
60 - "float32",
61 - "float64",
62 - "int32",
63 - "int8",
64 - "int4",
65 - "int2",
66 - "int1",
67 - "nf4",
68 - "fp4_e2m1",
69 - ],
70 - help="Data type of weight W.",
71 -)
72 -parser.add_argument(
73 - "--accum_dtype",
74 - type=str,
75 - default="float16",
76 - choices=["float16", "int32"],
77 - help="Data type for accumulation.",
78 -)
79 -parser.add_argument(
80 - "--out_dtype",
81 - type=str,
82 - default="float16",
83 - choices=["float16", "float32", "int32", "int8"],
84 - help="Data type for output.",
85 -)
86 -parser.add_argument(
87 - "--layout",
88 - type=str,
89 - default="nt",
90 - choices=["nt", "nn"],
91 - help="Matrix layout, 'nt' for non-transpose A and transpose W.",
92 -)
93 -parser.add_argument(
94 - "--with_bias", action="store_true", help="Include bias in the benchmark."
95 -)
96 -parser.add_argument(
97 - "--with_scaling",
98 - action="store_true",
99 - help="Include scaling factor in the quantization.",
100 -)
101 -parser.add_argument(
102 - "--with_zeros", action="store_true", help="Include zeros in the quantization."
103 -)
104 -parser.add_argument(
105 - "--zeros_mode",
106 - type=str,
107 - default=None,
108 - choices=["original", "rescale", "quantized"],
109 - help="Specify the mode for calculating zeros.",
110 -)
111 -
112 -# Parse the arguments
113 -args = parser.parse_args()
114 -
115 -# Assign arguments to variables
116 -target = args.target
117 -A_dtype = args.A_dtype
118 -W_dtype = args.W_dtype
119 -accum_dtype = args.accum_dtype
120 -out_dtype = args.out_dtype
121 -layout = args.layout
122 -with_bias = args.with_bias
123 -group_size = args.group_size
124 -with_scaling = args.with_scaling
125 -with_zeros = args.with_zeros
126 -zeros_mode = args.zeros_mode
127 -
128 -# Define a list of shared arguments that repeat in every config
129 -shared_args = [
130 - A_dtype,
131 - W_dtype,
132 - out_dtype,
133 - accum_dtype,
134 - layout,
135 - with_bias,
136 - group_size,
137 - with_scaling,
138 - with_zeros,
139 - zeros_mode,
140 -]
141 -
142 -# Define just the (M, K, N) shapes in a more compact list
143 -shapes = [
144 - # square test
145 - (1, 16384, 16384),
146 - # BLOOM-176B
147 - (1, 43008, 14336),
148 - (1, 14336, 14336),
149 - (1, 57344, 14336),
150 - (1, 14336, 57344),
151 - # OPT-65B
152 - (1, 9216, 9216),
153 - (1, 36864, 9216),
154 - (1, 9216, 36864),
155 - (1, 22016, 8192),
156 - # LLAMA-70B/65B
157 - (1, 8192, 22016),
158 - (1, 8192, 8192),
159 - (1, 28672, 8192),
160 - (1, 8192, 28672),
161 - # square test
162 - (16384, 16384, 16384),
163 - # BLOOM-176B
164 - (8192, 43008, 14336),
165 - (8192, 14336, 14336),
166 - (8192, 57344, 14336),
167 - (8192, 14336, 57344),
168 - # OPT-65B
169 - (8192, 9216, 9216),
170 - (8192, 36864, 9216),
171 - (8192, 9216, 36864),
172 - (8192, 22016, 8192),
173 - # LLAMA-70B/65B
174 - (8192, 8192, 22016),
175 - (8192, 8192, 8192),
176 - (8192, 28672, 8192),
177 - (8192, 8192, 28672),
178 -]
179 -
180 -# Build test shapes with all the shared arguments
181 -test_shapes = [(MatmulConfig, Matmul, (*shape, *shared_args)) for shape in shapes]
182 -
183 -benchmark_sets = []
184 -benchmark_sets.extend(test_shapes)
185 -
186 -benchmark_results = {}
187 -for config_class, operator, input_args in benchmark_sets:
188 - config = config_class(*input_args)
189 - matmul = operator(config, target=target, enable_tuning=True)
190 - kernel_latency = matmul.profile_latency()
191 -
192 - print("Time cost is: {:.3f} ms".format(kernel_latency))
193 -
194 - profile_config = {
195 - f"{operator.__name__}-{'-'.join([str(i) for i in input_args])}": {
196 - "BitBLAS_top20_latency": kernel_latency,
197 - }
198 - }
199 -
200 - benchmark_results.update(profile_config)
201 -
202 -# Define headers for the table
203 -headers = [
204 - "PrimFunc",
205 - "Input Arguments",
206 - "BitBLAS Top20 Latency",
207 -]
208 -
209 -# Calculate column widths for pretty printing
210 -col_widths = [0, 0, 0]
211 -for config_key, values in benchmark_results.items():
212 - args_split = config_key.split("-")
213 - func_name = args_split[0]
214 - input_args_str = "-".join(args_split[1:])
215 - col_widths[0] = max(col_widths[0], len(func_name) + 2, len(headers[0]) + 2)
216 - col_widths[1] = max(col_widths[1], len(input_args_str) + 2, len(headers[1]) + 2)
217 - col_widths[2] = max(
218 - col_widths[2],
219 - len(f"{values['BitBLAS_top20_latency']:.3f} ms") + 2,
220 - len(headers[2]) + 2,
221 - )
222 - # break only if you want to measure widths from a single example;
223 - # otherwise, let it loop over all items.
224 -
225 -# Print header
226 -for i, header in enumerate(headers):
227 - headers[i] = header.ljust(col_widths[i])
228 -print("".join(headers))
229 -print("-" * sum(col_widths))
230 -
231 -# Print rows
232 -for config_key, values in benchmark_results.items():
233 - args_split = config_key.split("-")
234 - func_name = args_split[0]
235 - input_args_str = "-".join(args_split[1:])
236 - row = [
237 - func_name,
238 - input_args_str,
239 - f"{values['BitBLAS_top20_latency']:.3f} ms",
240 - ]
241 - row_str = "".join(
242 - [str(cell).ljust(col_widths[idx]) for idx, cell in enumerate(row)]
243 - )
244 - print(row_str)
docs/features/quantization/README.md
-3
@@ -6,7 +6,6 @@ Contents:
6
7 - [AutoAWQ](auto_awq.md)
8 - [BitsAndBytes](bnb.md)
9 -- [BitBLAS](bitblas.md)
9 - [GGUF](gguf.md)
10 - [GPTQModel](gptqmodel.md)
11 - [Intel Neural Compressor](inc.md)
@@ -49,8 +48,6 @@ th:not(:first-child) {
48 | Marlin (GPTQ/AWQ/FP8) | ❌ | ❌ | ✅︎ | ✅︎ | ✅︎ | ❌ | ❌ | ❌ |
49 | INT8 (W8A8) | ❌ | ✅︎ | ✅︎ | ✅︎ | ✅︎ | ❌ | ❌ | ✅︎ |
50 | FP8 (W8A8) | ❌ | ❌ | ❌ | ✅︎ | ✅︎ | ✅︎ | ❌ | ❌ |
52 -| BitBLAS | ✅︎ | ✅ | ✅︎ | ✅︎ | ✅︎ | ❌ | ❌ | ❌ |
53 -| BitBLAS (GPTQ) | ❌ | ❌ | ✅︎ | ✅︎ | ✅︎ | ❌ | ❌ | ❌ |
51 | bitsandbytes | ✅︎ | ✅︎ | ✅︎ | ✅︎ | ✅︎ | ❌ | ❌ | ❌ |
52 | DeepSpeedFP | ✅︎ | ✅︎ | ✅︎ | ✅︎ | ✅︎ | ❌ | ❌ | ❌ |
53 | GGUF | ✅︎ | ✅︎ | ✅︎ | ✅︎ | ✅︎ | ✅︎ | ❌ | ❌ |
docs/features/quantization/bitblas.md deleted
-58
@@ -1,58 +0,0 @@
1 -# BitBLAS
2 -
3 -vLLM now supports [BitBLAS](https://github.com/microsoft/BitBLAS) for more efficient and flexible model inference. Compared to other quantization frameworks, BitBLAS provides more precision combinations.
4 -
5 -!!! note
6 - Ensure your hardware supports the selected `dtype` (`torch.bfloat16` or `torch.float16`).
7 - Most recent NVIDIA GPUs support `float16`, while `bfloat16` is more common on newer architectures like Ampere or Hopper.
8 - For details see [supported hardware](README.md#supported-hardware).
9 -
10 -Below are the steps to utilize BitBLAS with vLLM.
11 -
12 -```bash
13 -pip install bitblas>=0.1.0
14 -```
15 -
16 -vLLM reads the model's config file and supports pre-quantized checkpoints.
17 -
18 -You can find pre-quantized models on:
19 -
20 -- [Hugging Face (BitBLAS)](https://huggingface.co/models?search=bitblas)
21 -- [Hugging Face (GPTQ)](https://huggingface.co/models?search=gptq)
22 -
23 -Usually, these repositories have a `quantize_config.json` file that includes a `quantization_config` section.
24 -
25 -## Read bitblas format checkpoint
26 -
27 -```python
28 -from vllm import LLM
29 -import torch
30 -
31 -# "hxbgsyxh/llama-13b-4bit-g-1-bitblas" is a pre-quantized checkpoint.
32 -model_id = "hxbgsyxh/llama-13b-4bit-g-1-bitblas"
33 -llm = LLM(
34 - model=model_id,
35 - dtype=torch.bfloat16,
36 - trust_remote_code=True,
37 - quantization="bitblas",
38 -)
39 -```
40 -
41 -## Read gptq format checkpoint
42 -
43 -??? code
44 -
45 - ```python
46 - from vllm import LLM
47 - import torch
48 -
49 - # "hxbgsyxh/llama-13b-4bit-g-1" is a pre-quantized checkpoint.
50 - model_id = "hxbgsyxh/llama-13b-4bit-g-1"
51 - llm = LLM(
52 - model=model_id,
53 - dtype=torch.float16,
54 - trust_remote_code=True,
55 - quantization="bitblas",
56 - max_model_len=1024,
57 - )
58 - ```
tests/models/quantization/test_bitblas.py deleted
-68
@@ -1,68 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""Compare the outputs of a GPTQ model to a bitblas model.
4 -
5 -Note: GPTQ and bitblas do not have bitwise correctness.
6 -As a result, in this test, we just confirm that the top selected tokens of the
7 -bitblas/GPTQ models are in the top 3 selections of each other.
8 -
9 -Note: bitblas internally uses locks to synchronize the threads. This can
10 -result in very slight nondeterminism for bitblas. As a result, we re-run the
11 -test up to 3 times to see if we pass.
12 -"""
13 -
14 -from dataclasses import dataclass
15 -
16 -import pytest
17 -
18 -from ..utils import check_logprobs_close
19 -
20 -
21 -@dataclass
22 -class ModelPair:
23 - model_bitblas: str
24 - model_gptq: str
25 -
26 -
27 -model_pairs = [
28 - ModelPair(
29 - model_bitblas="hxbgsyxh/opt-125m-4bit-128g-bitblas",
30 - model_gptq="hxbgsyxh/opt-125m-4bit-128g",
31 - ),
32 -]
33 -
34 -
35 -@pytest.mark.flaky(reruns=2)
36 -@pytest.mark.skipif(True, reason="BitBLAS takes too much time for tuning.")
37 -@pytest.mark.parametrize("model_pair", model_pairs)
38 -@pytest.mark.parametrize("dtype", ["half"])
39 -@pytest.mark.parametrize("max_tokens", [32])
40 -@pytest.mark.parametrize("num_logprobs", [5])
41 -def test_models(
42 - vllm_runner,
43 - example_prompts,
44 - model_pair: ModelPair,
45 - dtype: str,
46 - max_tokens: int,
47 - num_logprobs: int,
48 -) -> None:
49 - with vllm_runner(
50 - model_pair.model_bitblas, dtype=dtype, quantization="bitblas"
51 - ) as bitblas_model:
52 - bitblas_outputs = bitblas_model.generate_greedy_logprobs(
53 - example_prompts, max_tokens, num_logprobs
54 - )
55 -
56 - with vllm_runner(
57 - model_pair.model_gptq, dtype=dtype, quantization="gptq"
58 - ) as gptq_model:
59 - gptq_outputs = gptq_model.generate_greedy_logprobs(
60 - example_prompts, max_tokens, num_logprobs
61 - )
62 -
63 - check_logprobs_close(
64 - outputs_0_lst=gptq_outputs,
65 - outputs_1_lst=bitblas_outputs,
66 - name_0="gptq",
67 - name_1="bitblas",
68 - )
tests/models/quantization/test_gptq_bitblas.py deleted
-64
@@ -1,64 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""Compare the outputs of a GPTQ model to a bitblas model.
4 -
5 -Note: GPTQ and bitblas do not have bitwise correctness.
6 -As a result, in this test, we just confirm that the top selected tokens of the
7 -bitblas/GPTQ models are in the top 3 selections of each other.
8 -
9 -Note: bitblas internally uses locks to synchronize the threads. This can
10 -result in very slight nondeterminism for bitblas. As a result, we re-run the
11 -test up to 3 times to see if we pass.
12 -"""
13 -
14 -from dataclasses import dataclass
15 -
16 -import pytest
17 -
18 -from ..utils import check_logprobs_close
19 -
20 -
21 -@dataclass
22 -class ModelPair:
23 - model_gptq: str
24 -
25 -
26 -model_pairs = [
27 - ModelPair(model_gptq="hxbgsyxh/opt-125m-4bit-128g"),
28 -]
29 -
30 -
31 -@pytest.mark.flaky(reruns=2)
32 -@pytest.mark.skipif(True, reason="BitBLAS takes too much time for tuning.")
33 -@pytest.mark.parametrize("model_pair", model_pairs)
34 -@pytest.mark.parametrize("dtype", ["half"])
35 -@pytest.mark.parametrize("max_tokens", [32])
36 -@pytest.mark.parametrize("num_logprobs", [5])
37 -def test_models(
38 - vllm_runner,
39 - example_prompts,
40 - model_pair: ModelPair,
41 - dtype: str,
42 - max_tokens: int,
43 - num_logprobs: int,
44 -) -> None:
45 - with vllm_runner(
46 - model_pair.model_gptq, dtype=dtype, quantization="bitblas"
47 - ) as bitblas_model:
48 - bitblas_outputs = bitblas_model.generate_greedy_logprobs(
49 - example_prompts, max_tokens, num_logprobs
50 - )
51 -
52 - with vllm_runner(
53 - model_pair.model_gptq, dtype=dtype, quantization="gptq"
54 - ) as gptq_model:
55 - gptq_outputs = gptq_model.generate_greedy_logprobs(
56 - example_prompts, max_tokens, num_logprobs
57 - )
58 -
59 - check_logprobs_close(
60 - outputs_0_lst=gptq_outputs,
61 - outputs_1_lst=bitblas_outputs,
62 - name_0="gptq",
63 - name_1="gptq_bitblas",
64 - )
vllm/config/model.py
-2
@@ -890,10 +890,8 @@ class ModelConfig:
890 # `override_quantization_method` method) must be checked in order
891 # of preference (this is particularly important for GPTQ).
892 overrides = [
893 - "bitblas",
893 "gptq_marlin_24",
894 "gptq_marlin",
896 - "gptq_bitblas",
895 "awq_marlin",
896 "ipex",
897 "inc",
vllm/model_executor/layers/linear.py
-17
@@ -41,8 +41,6 @@ WEIGHT_LOADER_V2_SUPPORTED = [
41 "UnquantizedLinearMethod",
42 "CompressedTensorsLinearMethod",
43 "CompressedTensorsLinearTransformMethod",
44 - "BitBLASLinearMethod",
45 - "GPTQBitBLASLinearMethod",
44 "AWQMarlinLinearMethod",
45 "AWQLinearMethod",
46 "GPTQMarlinLinearMethod",
@@ -63,14 +61,6 @@ WEIGHT_LOADER_V2_SUPPORTED = [
61 ]
62
63
66 -def adjust_bitblas_shard(param, shard_size, shard_offset):
67 - bitblas_tile_size = getattr(param, "bitblas_tile_size", None)
68 - if bitblas_tile_size is not None:
69 - return (shard_size // bitblas_tile_size, shard_offset // bitblas_tile_size)
70 -
71 - return shard_size, shard_offset
72 -
73 -
64 def adjust_marlin_shard(param, shard_size, shard_offset):
65 marlin_tile_size = getattr(param, "marlin_tile_size", None)
66 if marlin_tile_size is None:
@@ -748,10 +738,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
738 param, shard_size, shard_offset
739 )
740
751 - shard_size, shard_offset = adjust_bitblas_shard(
752 - param, shard_size, shard_offset
753 - )
754 -
741 if use_bitsandbytes_4bit:
742 index = list(itertools.accumulate([0] + self.output_sizes))
743 orig_offsets = {
@@ -794,9 +780,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
780 shard_size, shard_offset = adjust_marlin_shard(
781 param, shard_size, shard_offset
782 )
797 - shard_size, shard_offset = adjust_bitblas_shard(
798 - param, shard_size, shard_offset
799 - )
783
784 use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
785 is_sharded_weight = getattr(param, "is_sharded_weight", False)
vllm/model_executor/layers/quantization/__init__.py
-8
@@ -17,11 +17,9 @@ QuantizationMethods = Literal[
17 "fp_quant",
18 "modelopt",
19 "modelopt_fp4",
20 - "bitblas",
20 "gguf",
21 "gptq_marlin_24",
22 "gptq_marlin",
24 - "gptq_bitblas",
23 "awq_marlin",
24 "gptq",
25 "compressed-tensors",
@@ -43,9 +41,7 @@ DEPRECATED_QUANTIZATION_METHODS = [
41 "ptpc_fp8",
42 "fbgemm_fp8",
43 "fp_quant",
46 - "bitblas",
44 "gptq_marlin_24",
48 - "gptq_bitblas",
45 "experts_int8",
46 "ipex",
47 "petit_nvfp4",
@@ -114,7 +110,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
110
111 from .awq import AWQConfig
112 from .awq_marlin import AWQMarlinConfig
117 - from .bitblas import BitBLASConfig
113 from .bitsandbytes import BitsAndBytesConfig
114 from .compressed_tensors.compressed_tensors import (
115 CompressedTensorsConfig,
@@ -126,7 +121,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
121 from .fp_quant import FPQuantConfig
122 from .gguf import GGUFConfig
123 from .gptq import GPTQConfig
129 - from .gptq_bitblas import GPTQBitBLASConfig
124 from .gptq_marlin import GPTQMarlinConfig
125 from .gptq_marlin_24 import GPTQMarlin24Config
126 from .inc import INCConfig
@@ -145,11 +139,9 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
139 "fp_quant": FPQuantConfig,
140 "modelopt": ModelOptFp8Config,
141 "modelopt_fp4": ModelOptNvFp4Config,
148 - "bitblas": BitBLASConfig,
142 "gguf": GGUFConfig,
143 "gptq_marlin_24": GPTQMarlin24Config,
144 "gptq_marlin": GPTQMarlinConfig,
152 - "gptq_bitblas": GPTQBitBLASConfig,
145 "awq_marlin": AWQMarlinConfig,
146 "gptq": GPTQConfig,
147 "compressed-tensors": CompressedTensorsConfig,
vllm/model_executor/layers/quantization/bitblas.py deleted
-502
@@ -1,502 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -from typing import Any, Optional
4 -
5 -import torch
6 -from packaging import version
7 -
8 -from vllm.logger import init_logger
9 -from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
10 -from vllm.model_executor.layers.quantization import (
11 - QuantizationConfig,
12 - QuantizationMethods,
13 -)
14 -from vllm.model_executor.layers.quantization.utils.bitblas_utils import (
15 - BITBLAS_OPTIMIZE_FEATURES,
16 - BITBLAS_SUPPORTED_NUM_BITS,
17 - BITBLAS_SUPPORTED_SYM,
18 - MINIMUM_BITBLAS_VERSION,
19 -)
20 -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
21 -from vllm.model_executor.parameter import (
22 - BasevLLMParameter,
23 - ChannelQuantScaleParameter,
24 - GroupQuantScaleParameter,
25 - PackedvLLMParameter,
26 -)
27 -from vllm.model_executor.utils import set_weight_attrs
28 -
29 -logger = init_logger(__name__)
30 -
31 -
32 -class BitBLASConfig(QuantizationConfig):
33 - """Config class for BitBLAS.
34 -
35 - Reference: https://github.com/Microsoft/BitBLAS
36 - """
37 -
38 - TORCH_DTYPE = torch.float16
39 - STORAGE_DTYPE = "int8" # assume int8 storage
40 - TORCH_STORAGE_DTYPE = getattr(torch, STORAGE_DTYPE)
41 - # "original" or "rescale" or "quantized",
42 - # gptq_with_bitblas prefer "quantized implementation"
43 - ZEROS_MODE = "quantized"
44 -
45 - def __init__(
46 - self,
47 - weight_bits: int,
48 - group_size: int | None,
49 - desc_act: bool | None,
50 - is_sym: bool | None,
51 - quant_method: str | None,
52 - lm_head_quantized: bool,
53 - ) -> None:
54 - try:
55 - import bitblas
56 -
57 - if version.parse(bitblas.__version__) < version.parse(
58 - MINIMUM_BITBLAS_VERSION
59 - ):
60 - raise ImportError(
61 - "bitblas version is wrong. Please "
62 - f"install bitblas>={MINIMUM_BITBLAS_VERSION}"
63 - )
64 - except ImportError as e:
65 - bitblas_import_exception = e
66 - raise ValueError(
67 - "Trying to use the bitblas backend, but could not import"
68 - f"with the following error: {bitblas_import_exception}. "
69 - "Please install bitblas through the following command: "
70 - f"`pip install bitblas>={MINIMUM_BITBLAS_VERSION}`"
71 - ) from bitblas_import_exception
72 -
73 - if desc_act and group_size == -1:
74 - # In this case, act_order == True is the same as act_order == False
75 - # (since we have only one group per output channel)
76 - desc_act = False
77 -
78 - super().__init__()
79 - self.weight_bits = weight_bits
80 - self.group_size = group_size
81 - self.desc_act = desc_act
82 - self.is_sym = is_sym
83 - self.quant_method = quant_method
84 - self.lm_head_quantized = lm_head_quantized
85 -
86 - # Verify
87 - if self.weight_bits not in BITBLAS_SUPPORTED_NUM_BITS:
88 - raise ValueError(
89 - f"BitBLAS does not support weight_bits = {self.weight_bits}. "
90 - f"Only weight_bits = {BITBLAS_SUPPORTED_NUM_BITS} "
91 - "are supported."
92 - )
93 -
94 - if self.is_sym not in BITBLAS_SUPPORTED_SYM:
95 - raise ValueError(
96 - f"BitBLAS does not support is_sym = {self.is_sym}. "
97 - f"Only sym = {BITBLAS_SUPPORTED_SYM} are supported."
98 - )
99 -
100 - storage_dtype = self.STORAGE_DTYPE
101 - storage_nbit = int("".join(c for c in storage_dtype if c.isdigit()))
102 -
103 - self.storage_dtype = storage_dtype
104 - self.storage_torch_dtype = self.TORCH_STORAGE_DTYPE
105 - # 4 Bits packed into 32 bit datatype.
106 - self.pack_factor = storage_nbit // weight_bits
107 - self.nbits = weight_bits
108 -
109 - # Zeros type for the quantized weights.
110 - self.zeros_mode = self.ZEROS_MODE
111 -
112 - def __repr__(self) -> str:
113 - return (
114 - f"BitBLASConfig(weight_bits={self.weight_bits}, "
115 - f"group_size={self.group_size}, "
116 - f"desc_act={self.desc_act}, "
117 - f"is_sym={self.is_sym}, "
118 - f"quant_method={self.quant_method})"
119 - )
120 -
121 - @classmethod
122 - def get_name(cls) -> QuantizationMethods:
123 - return "bitblas"
124 -
125 - @classmethod
126 - def get_supported_act_dtypes(cls) -> list[torch.dtype]:
127 - return [torch.half, torch.bfloat16]
128 -
129 - @classmethod
130 - # Need to figure it out
131 - def get_min_capability(cls) -> int:
132 - return 70
133 -
134 - @classmethod
135 - def get_config_filenames(cls) -> list[str]:
136 - return ["quantize_config.json"]
137 -
138 - @staticmethod
139 - def get_from_keys(
140 - config: dict[str, Any], keys: list[str], default: Any = None
141 - ) -> Any:
142 - """Get a value from the model's quantization config."""
143 - for key in keys:
144 - if key in config:
145 - return config[key]
146 - return default
147 -
148 - @classmethod
149 - def from_config(cls, config: dict[str, Any]) -> "BitBLASConfig":
150 - weight_bits = cls.get_from_keys(config, ["bits"])
151 - group_size = cls.get_from_keys(config, ["group_size"], -1)
152 - desc_act = cls.get_from_keys(config, ["desc_act"], False)
153 - is_sym = cls.get_from_keys(config, ["sym"], False)
154 - quant_method = cls.get_from_keys(config, ["quant_method"])
155 - lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False)
156 - return cls(
157 - weight_bits, group_size, desc_act, is_sym, quant_method, lm_head_quantized
158 - )
159 -
160 - @classmethod
161 - def override_quantization_method(
162 - cls, hf_quant_cfg, user_quant
163 - ) -> QuantizationMethods | None:
164 - # compat: autogptq >=0.8.0 use checkpoint_format: str
165 - # compat: autogptq <=0.7.1 is_bitblas_format: bool
166 - is_bitblas_format = hf_quant_cfg.get(
167 - "checkpoint_format"
168 - ) == "bitblas" or hf_quant_cfg.get("is_bitblas_format", False)
169 -
170 - is_valid_user_quant = (
171 - user_quant is None or user_quant == "gptq" or user_quant == "bitblas"
172 - )
173 -
174 - if is_bitblas_format and is_valid_user_quant:
175 - msg = "The model is serialized in {} format. Using {} kernel.".format(
176 - cls.get_name(), cls.get_name()
177 - )
178 - logger.info(msg)
179 - return cls.get_name()
180 -
181 - return None
182 -
183 - def get_quant_method(
184 - self, layer: torch.nn.Module, prefix: str
185 - ) -> Optional["BitBLASLinearMethod"]:
186 - if isinstance(layer, LinearBase) or (
187 - isinstance(layer, ParallelLMHead) and self.lm_head_quantized
188 - ):
189 - return BitBLASLinearMethod(self)
190 - return None
191 -
192 -
193 -class BitBLASLinearMethod(LinearMethodBase):
194 - """Linear method for BitBLAS.
195 -
196 - Args:
197 - quant_config: The BitBLAS quantization config.
198 - """
199 -
200 - # USE BITBLAS_OPTIMIZE_FEATURES_CONTIGUOUS
201 - # Instead of BITBLAS_OPTIMIZE_FEATURES
202 - # If you want to high contiguous batching
203 - # performance
204 - OPT_FEATURES = BITBLAS_OPTIMIZE_FEATURES
205 - ENABLE_TUNING = True
206 - BITBLAS_DTYPES = {
207 - torch.float32: "float32",
208 - torch.float16: "float16",
209 - torch.bfloat16: "bfloat16",
210 - torch.half: "float16",
211 - torch.int8: "int8",
212 - }
213 -
214 - def __init__(self, quant_config: BitBLASConfig):
215 - self.quant_config = quant_config
216 -
217 - def create_weights_gptq(
218 - self,
219 - layer: torch.nn.Module,
220 - input_size_per_partition: int,
221 - output_partition_sizes: list[int],
222 - input_size: int,
223 - output_size: int,
224 - params_dtype: torch.dtype,
225 - **extra_weight_attrs,
226 - ) -> None:
227 - """Creates quantized weights for use in linear operations.
228 -
229 - The function initializes and returns a dictionary containing quantized
230 - weights, scales, and zeros
231 - for performing quantized matrix multiplication operations.
232 -
233 - Args:
234 - input_size_per_partition: The size of the input partition.
235 - output_partition_sizes: List of output partition sizes.
236 - input_size: The total size of the input (unused).
237 - output_size: The total size of the output (unused).
238 - params_dtype:
239 - The data type of the parameters (expected to be torch.float16).
240 -
241 - Returns:
242 - A dictionary containing the quantized weights ('qweight'),
243 - scales ('scales'), and zeros ('zeros').
244 -
245 - Raises:
246 - ValueError: If `params_dtype` is not `torch.float16` or if the input
247 - size per partition is not divisible by the group size
248 - in `quant_config`.
249 - """
250 - del input_size, output_size # Unused arguments.
251 - weight_loader = extra_weight_attrs["weight_loader"]
252 -
253 - if params_dtype not in self.quant_config.get_supported_act_dtypes():
254 - raise ValueError(
255 - f"Parameter data type must be torch.float16, but got {params_dtype}"
256 - )
257 - group_size = self.quant_config.group_size
258 - if group_size is None:
259 - group_size = -1
260 - # Validate output_size_per_partition
261 - output_size_per_partition = sum(output_partition_sizes)
262 - if group_size != -1 and input_size_per_partition % group_size != 0:
263 - raise ValueError(
264 - f"Input size per partition ({input_size_per_partition}) must "
265 - f"be divisible by group size ({group_size})."
266 - )
267 -
268 - # Initialize or retrieve the BitBLAS matrix multiplication operator.
269 - self._configure_bitblas_matmul(
270 - input_size_per_partition,
271 - output_size_per_partition,
272 - params_dtype=params_dtype,
273 - enable_tuning=self.ENABLE_TUNING,
274 - bias=False,
275 - layout="nt",
276 - bits=self.quant_config.weight_bits,
277 - )
278 -
279 - # Initialize quantized weights with dimensions
280 - # Quantized 4Bit weights packed.
281 - qweight = PackedvLLMParameter(
282 - data=torch.empty(
283 - self.bitblas_matmul.retrieve_weight_shape(),
284 - device="cuda",
285 - dtype=self.quant_config.storage_torch_dtype,
286 - requires_grad=False,
287 - ),
288 - input_dim=1,
289 - output_dim=0,
290 - packed_dim=1,
291 - packed_factor=self.quant_config.pack_factor,
292 - bitblas_tile_size=(
293 - self.bitblas_matmul.retrieve_weight_shape()[-2]
294 - if self.bitblas_matmul.propagate_b
295 - else None
296 - ),
297 - weight_loader=weight_loader,
298 - )
299 -
300 - # Compute the number of input groups for channel-wise quantization.
301 - input_groups = 1 if group_size == -1 else input_size_per_partition // group_size
302 -
303 - # Initialize scales and zeros for the quantized weights.
304 - weight_scale_args = {
305 - "data": torch.empty(
306 - output_size_per_partition,
307 - input_groups,
308 - device="cuda",
309 - dtype=params_dtype,
310 - ),
311 - "weight_loader": weight_loader,
312 - }
313 - if input_groups == 1:
314 - scales = ChannelQuantScaleParameter(output_dim=0, **weight_scale_args)
315 - else:
316 - scales = GroupQuantScaleParameter(
317 - output_dim=0, input_dim=1, **weight_scale_args
318 - )
319 -
320 - if self.quant_config.zeros_mode == "quantized":
321 - zeros = PackedvLLMParameter(
322 - data=torch.empty(
323 - input_groups,
324 - output_size_per_partition // self.quant_config.pack_factor,
325 - device="cuda",
326 - dtype=self.quant_config.storage_torch_dtype,
327 - requires_grad=False,
328 - ),
329 - input_dim=0,
330 - output_dim=1,
331 - packed_dim=1,
332 - packed_factor=self.quant_config.pack_factor,
333 - weight_loader=weight_loader,
334 - )
335 -
336 - else:
337 - zeros = BasevLLMParameter(
338 - torch.empty(
339 - output_size_per_partition,
340 - input_groups,
341 - device="cuda",
342 - dtype=params_dtype,
343 - ),
344 - weight_loader=weight_loader,
345 - )
346 - # Set attributes to indicate how scales and zeros are applied.
347 - set_weight_attrs(
348 - zeros,
349 - {
350 - "input_dim": None if input_groups == 1 else 1,
351 - "output_dim": 0,
352 - },
353 - )
354 -
355 - layer.register_parameter("qweight", qweight)
356 - layer.register_parameter("scales", scales)
357 - layer.register_parameter("zeros", zeros)
358 -
359 - def create_weights(
360 - self,
361 - layer: torch.nn.Module,
362 - input_size_per_partition: int,
363 - output_partition_sizes: list[int],
364 - input_size: int,
365 - output_size: int,
366 - params_dtype: torch.dtype,
367 - **extra_weight_attrs,
368 - ):
369 - if self.quant_config.quant_method == "gptq":
370 - return self.create_weights_gptq(
371 - layer,
372 - input_size_per_partition,
373 - output_partition_sizes,
374 - input_size,
375 - output_size,
376 - params_dtype,
377 - **extra_weight_attrs,
378 - )
379 - else:
380 - raise ValueError(
381 - f"Unsupported quant_method {self.quant_config.quant_method}"
382 - )
383 -
384 - def _configure_bitblas_matmul(
385 - self,
386 - infeatures,
387 - outfeatures,
388 - params_dtype,
389 - enable_tuning,
390 - bias,
391 - layout,
392 - bits,
393 - out_dtype="float16",
394 - ):
395 - from bitblas import MatmulConfig
396 -
397 - bitblas_dtype = self.BITBLAS_DTYPES[params_dtype]
398 -
399 - with_scaling = False
400 - with_zeros = False
401 - group_size = self.quant_config.group_size
402 - zeros_mode = self.quant_config.zeros_mode
403 - if self.quant_config.quant_method == "gptq":
404 - with_scaling = True
405 - with_zeros = True
406 - W_dtype = f"uint{bits}"
407 - if self.quant_config.is_sym:
408 - with_zeros = False
409 - W_dtype = f"int{bits}"
410 - else:
411 - raise ValueError(
412 - f"Unsupported quant_method {self.quant_config.quant_method}"
413 - )
414 -
415 - matmul_config = MatmulConfig(
416 - N=outfeatures,
417 - K=infeatures,
418 - A_dtype=bitblas_dtype,
419 - W_dtype=W_dtype,
420 - out_dtype=out_dtype,
421 - accum_dtype="int32" if bitblas_dtype == "int8" else bitblas_dtype,
422 - storage_dtype=self.quant_config.STORAGE_DTYPE,
423 - with_scaling=with_scaling,
424 - with_zeros=with_zeros,
425 - group_size=group_size,
426 - with_bias=bias,
427 - layout=layout,
428 - zeros_mode=zeros_mode,
429 - )
430 - self.bitblas_matmul = self._get_or_create_bitblas_operator(
431 - matmul_config, enable_tuning
432 - )
433 -
434 - def _get_or_create_bitblas_operator(self, config, enable_tuning):
435 - from bitblas import Matmul, auto_detect_nvidia_target
436 - from bitblas.cache import get_database_path, global_operator_cache
437 -
438 - BITBLAS_DATABASE_PATH = get_database_path()
439 - BITBLAS_TARGET = auto_detect_nvidia_target()
440 - if global_operator_cache.size() == 0:
441 - global_operator_cache.load_from_database(
442 - BITBLAS_DATABASE_PATH, BITBLAS_TARGET
443 - )
444 -
445 - bitblas_matmul = global_operator_cache.get(config)
446 - if bitblas_matmul is None:
447 - bitblas_matmul = Matmul(config, target=BITBLAS_TARGET, enable_tuning=False)
448 - if enable_tuning:
449 - TUNING_MESSAGE = f"BitBLAS Operator {config} is tuning ..."
450 - logger.info(TUNING_MESSAGE)
451 - bitblas_matmul.hardware_aware_finetune(topk=20)
452 - global_operator_cache.add(config, bitblas_matmul)
453 - global_operator_cache.save_into_database(
454 - BITBLAS_DATABASE_PATH, BITBLAS_TARGET
455 - )
456 - TUNED_MESSAGE = (
457 - f"BitBLAS Operator {config} tuned and saved to database."
458 - )
459 - logger.info(TUNED_MESSAGE)
460 - else:
461 - _message = f"BitBLAS Operator {config} created."
462 - logger.info(_message)
463 - else:
464 - _message = f"BitBLAS Operator {config} found in global_operator_cache."
465 - logger.info(_message)
466 - return bitblas_matmul
467 -
468 - def apply_gptq(
469 - self,
470 - layer: torch.nn.Module,
471 - x: torch.Tensor,
472 - bias: torch.Tensor | None = None,
473 - ) -> torch.Tensor:
474 - qweight = layer.qweight
475 - scales = layer.scales
476 - qzeros = layer.zeros
477 -
478 - x_2d = x.view(-1, x.shape[-1])
479 -
480 - if self.quant_config.is_sym:
481 - output_2d = self.bitblas_matmul(x_2d, qweight, scales)
482 - else:
483 - output_2d = self.bitblas_matmul(x_2d, qweight, scales, qzeros)
484 -
485 - output = output_2d.view(x.shape[:-1] + (output_2d.shape[1],))
486 -
487 - if bias is not None:
488 - output.add_(bias) # In-place add
489 -
490 - return output
491 -
492 - def apply(
493 - self,
494 - *args: Any,
495 - **kwargs: Any,
496 - ) -> torch.Tensor:
497 - if self.quant_config.quant_method == "gptq":
498 - return self.apply_gptq(*args, **kwargs)
499 - else:
500 - raise ValueError(
501 - f"Unsupported quant_method {self.quant_config.quant_method}"
502 - )
vllm/model_executor/layers/quantization/gptq.py
+1 -1
@@ -98,7 +98,7 @@ class GPTQConfig(QuantizationConfig):
98 if self.weight_bits == 4:
99 logger.warning_once(
100 "Currently, the 4-bit gptq_gemm kernel for GPTQ is buggy. "
101 - "Please switch to gptq_marlin or gptq_bitblas."
101 + "Please switch to gptq_marlin."
102 )
103
104 self.modules_in_block_to_quantize = modules_in_block_to_quantize or []
vllm/model_executor/layers/quantization/gptq_bitblas.py deleted
-482
@@ -1,482 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -from typing import Any, Optional
4 -
5 -import torch
6 -from packaging import version
7 -from torch.nn.parameter import Parameter
8 -
9 -from vllm.logger import init_logger
10 -from vllm.model_executor.layers.linear import (
11 - LinearBase,
12 - LinearMethodBase,
13 - set_weight_attrs,
14 -)
15 -from vllm.model_executor.layers.quantization import (
16 - QuantizationConfig,
17 - QuantizationMethods,
18 -)
19 -from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
20 - BitBLASLinearKernel,
21 - MPLinearLayerConfig,
22 -)
23 -from vllm.model_executor.layers.quantization.utils.bitblas_utils import (
24 - BITBLAS_SUPPORTED_NUM_BITS as GPTQ_BITBLAS_SUPPORTED_NUM_BITS,
25 -)
26 -from vllm.model_executor.layers.quantization.utils.bitblas_utils import (
27 - BITBLAS_SUPPORTED_SYM as GPTQ_BITBLAS_SUPPORTED_SYM,
28 -)
29 -from vllm.model_executor.layers.quantization.utils.bitblas_utils import (
30 - MINIMUM_BITBLAS_VERSION,
31 - bitblas_repeat_scales_on_all_ranks,
32 - check_bitblas_supported,
33 - verify_bitblas_supported,
34 -)
35 -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
36 -from vllm.model_executor.parameter import (
37 - ChannelQuantScaleParameter,
38 - GroupQuantScaleParameter,
39 - PackedColumnParameter,
40 - PackedvLLMParameter,
41 - RowvLLMParameter,
42 -)
43 -from vllm.platforms import current_platform
44 -from vllm.scalar_type import scalar_types
45 -
46 -logger = init_logger(__name__)
47 -
48 -
49 -class GPTQBitBLASConfig(QuantizationConfig):
50 - """Config class for GPTQ BitBLAS"""
51 -
52 - # (num_bits, is_sym) -> quant_type
53 - TYPE_MAP = {
54 - (4, True): scalar_types.uint4b8,
55 - (8, True): scalar_types.uint8b128,
56 - }
57 -
58 - TORCH_DTYPE = torch.float16
59 - GPTQ_CKPT_STORAGE_DTYPE = (
60 - "int32" # GPTQ Default Checkpoints use int32 as storage dtype
61 - )
62 - GPTQ_BITBLAS_STORAGE_DTYPE = "int8" # BitBLAS uses int8 as storage dtype
63 - TORCH_BITBLAS_STORAGE_DTYPE = getattr(torch, GPTQ_BITBLAS_STORAGE_DTYPE)
64 - # "original" or "rescale" or "quantized",
65 - # the gptq_bitblas prefer "quantized"
66 - ZEROS_MODE = "quantized"
67 -
68 - def __init__(
69 - self,
70 - weight_bits: int,
71 - group_size: int,
72 - desc_act: bool,
73 - is_sym: bool,
74 - quant_method: str | None,
75 - lm_head_quantized: bool,
76 - ) -> None:
77 - try:
78 - import bitblas
79 -
80 - if version.parse(bitblas.__version__) < version.parse(
81 - MINIMUM_BITBLAS_VERSION
82 - ):
83 - raise ImportError(
84 - "bitblas version is wrong. Please "
85 - f"install bitblas>={MINIMUM_BITBLAS_VERSION}"
86 - )
87 - except ImportError as e:
88 - bitblas_import_exception = e
89 - raise ValueError(
90 - "Trying to use the bitblas backend, but could not import"
91 - f"with the following error: {bitblas_import_exception}. "
92 - "Please install bitblas through the following command: "
93 - f"`pip install bitblas>={MINIMUM_BITBLAS_VERSION}`"
94 - ) from bitblas_import_exception
95 -
96 - if desc_act and group_size == -1:
97 - # In this case, act_order == True is the same as act_order == False
98 - # (since we have only one group per output channel)
99 - desc_act = False
100 -
101 - super().__init__()
102 - self.weight_bits = weight_bits
103 - self.group_size = group_size
104 - self.desc_act = desc_act
105 - self.is_sym = is_sym
106 - self.quant_method = quant_method
107 - self.lm_head_quantized = lm_head_quantized
108 -
109 - # Verify
110 - if self.weight_bits not in GPTQ_BITBLAS_SUPPORTED_NUM_BITS:
111 - raise ValueError(
112 - f"BitBLAS does not support weight_bits = {self.weight_bits}. "
113 - f"Only weight_bits = {GPTQ_BITBLAS_SUPPORTED_NUM_BITS} "
114 - "are supported."
115 - )
116 -
117 - if self.is_sym not in GPTQ_BITBLAS_SUPPORTED_SYM:
118 - raise ValueError(
119 - f"BitBLAS does not support is_sym = {self.is_sym}. "
120 - f"Only sym = {GPTQ_BITBLAS_SUPPORTED_SYM} are supported."
121 - )
122 -
123 - self.storage_dtype = self.GPTQ_BITBLAS_STORAGE_DTYPE
124 -
125 - storage_nbit = int(
126 - "".join(c for c in self.GPTQ_CKPT_STORAGE_DTYPE if c.isdigit())
127 - )
128 -
129 - # 4 Bits packed into 32 bit datatype.
130 - self.pack_factor = storage_nbit // weight_bits
131 - self.nbits = weight_bits
132 -
133 - # Zeros type for the quantized weights.
134 - self.zeros_mode = self.ZEROS_MODE
135 -
136 - if (weight_bits, is_sym) not in self.TYPE_MAP:
137 - raise ValueError(
138 - f"Unsupported quantization config: bits={weight_bits}, sym={is_sym}"
139 - )
140 -
141 - self.quant_type = self.TYPE_MAP[(weight_bits, is_sym)]
142 -
143 - def __repr__(self) -> str:
144 - return (
145 - f"GPTQBitBLASConfig(weight_bits={self.weight_bits}, "
146 - f"group_size={self.group_size}, "
147 - f"desc_act={self.desc_act})"
148 - f"is_sym={self.is_sym}, "
149 - f"quant_method={self.quant_method})"
150 - )
151 -
152 - @classmethod
153 - def get_name(cls) -> QuantizationMethods:
154 - return "gptq_bitblas"
155 -
156 - @classmethod
157 - def get_supported_act_dtypes(cls) -> list[torch.dtype]:
158 - return [torch.half, torch.bfloat16]
159 -
160 - @classmethod
161 - def get_min_capability(cls) -> int:
162 - return 80
163 -
164 - @classmethod
165 - def get_config_filenames(cls) -> list[str]:
166 - return ["quantize_config.json"]
167 -
168 - @classmethod
169 - def from_config(cls, config: dict[str, Any]) -> "GPTQBitBLASConfig":
170 - weight_bits = cls.get_from_keys(config, ["bits"])
171 - group_size = cls.get_from_keys(config, ["group_size"])
172 - desc_act = cls.get_from_keys(config, ["desc_act"])
173 - is_sym = cls.get_from_keys(config, ["sym"])
174 - quant_method = cls.get_from_keys(config, ["quant_method"])
175 - lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False)
176 - return cls(
177 - weight_bits, group_size, desc_act, is_sym, quant_method, lm_head_quantized
178 - )
179 -
180 - @classmethod
181 - def override_quantization_method(
182 - cls, hf_quant_cfg, user_quant
183 - ) -> QuantizationMethods | None:
184 - can_convert = cls.is_gptq_bitblas_compatible(hf_quant_cfg)
185 -
186 - is_valid_user_quant = (
187 - user_quant is None
188 - or user_quant == "bitblas"
189 - or user_quant == "gptq_bitblas"
190 - )
191 -
192 - if can_convert and is_valid_user_quant:
193 - msg = (
194 - "The model is convertible to {} during runtime."
195 - " Using {} kernel.".format(cls.get_name(), cls.get_name())
196 - )
197 - logger.info(msg)
198 - return cls.get_name()
199 -
200 - if can_convert and user_quant == "gptq":
201 - logger.info(
202 - "Detected that the model can run with gptq_bitblas"
203 - ", however you specified quantization=gptq explicitly,"
204 - " so forcing gptq. Use quantization=gptq_bitblas for"
205 - " faster inference"
206 - )
207 - return None
208 -
209 - def get_quant_method(
210 - self, layer: torch.nn.Module, prefix: str
211 - ) -> Optional["GPTQBitBLASLinearMethod"]:
212 - if isinstance(layer, LinearBase) or (
213 - isinstance(layer, ParallelLMHead) and self.lm_head_quantized
214 - ):
215 - return GPTQBitBLASLinearMethod(self)
216 - return None
217 -
218 - @property
219 - def torch_storage_dtype(self) -> torch.dtype:
220 - return self.TORCH_BITBLAS_STORAGE_DTYPE
221 -
222 - @classmethod
223 - def is_gptq_bitblas_compatible(cls, quant_config: dict[str, Any]):
224 - # Extract data from quant config.
225 - num_bits = quant_config.get("bits")
226 - group_size = quant_config.get("group_size")
227 - sym = quant_config.get("sym")
228 - desc_act = quant_config.get("desc_act")
229 -
230 - # temporarily disable on ROCm platform
231 - if not current_platform.is_cuda():
232 - return False
233 -
234 - # If we cannot find the info needed in the config, cannot convert.
235 - if num_bits is None or group_size is None or sym is None or desc_act is None:
236 - return False
237 -
238 - if (num_bits, sym) not in cls.TYPE_MAP:
239 - return False
240 -
241 - # If the capability of the device is too low, cannot convert.
242 - major, minor = torch.cuda.get_device_capability()
243 - device_capability = major * 10 + minor
244 - if device_capability < cls.get_min_capability():
245 - return False
246 -
247 - # Otherwise, can convert if model satisfies bitblas constraints.
248 - return check_bitblas_supported(
249 - quant_type=cls.TYPE_MAP[(num_bits, sym)], group_size=group_size
250 - )
251 -
252 -
253 -class GPTQBitBLASLinearMethod(LinearMethodBase):
254 - """Linear method for GPTQ BitBLAS.
255 -
256 - Args:
257 - quant_config: The GPTQ BitBLAS quantization config.
258 - """
259 -
260 - kernel_type = BitBLASLinearKernel
261 - _kernel_backends_being_used: set[str] = set()
262 -
263 - def __init__(self, quant_config: GPTQBitBLASConfig) -> None:
264 - self.quant_config = quant_config
265 - # Verify supported on platform.
266 - verify_bitblas_supported(
267 - quant_type=self.quant_config.quant_type,
268 - group_size=self.quant_config.group_size,
269 - )
270 -
271 - def create_weights(
272 - self,
273 - layer: torch.nn.Module,
274 - input_size_per_partition: int,
275 - output_partition_sizes: list[int],
276 - input_size: int,
277 - output_size: int,
278 - params_dtype: torch.dtype,
279 - **extra_weight_attrs,
280 - ) -> None:
281 - """Creates quantized weights for use in linear operations.
282 -
283 - The function initializes and returns a dictionary containing
284 - quantized weights, scales, and zeros
285 - for performing quantized matrix multiplication operations.
286 -
287 - Args:
288 - input_size_per_partition: The size of the input partition.
289 - output_partition_sizes: The size of the output partition.
290 - input_size: The total size of the input (unused).
291 - output_size: The total size of the output (unused).
292 - params_dtype:
293 - The data type of the parameters (expected to be torch.float16).
294 -
295 - Returns:
296 - A dictionary containing the quantized weights ('qweight'),
297 - scales ('scales'), and zeros ('zeros').
298 -
299 - Raises:
300 - ValueError: If `params_dtype` is not `torch.float16` or if the input
301 - size per partition is not divisible by the group size
302 - in `quant_config`.
303 - """
304 - if params_dtype != torch.float16:
305 - raise ValueError(
306 - f"Parameter data type must be torch.float16, but got {params_dtype}"
307 - )
308 -
309 - # Normalize group_size
310 - if self.quant_config.group_size != -1:
311 - group_size = self.quant_config.group_size
312 - else:
313 - group_size = input_size
314 -
315 - if input_size_per_partition % group_size != 0:
316 - raise ValueError(
317 - f"Input size per partition ({input_size_per_partition}) must "
318 - f"be divisible by group size ({self.quant_config.group_size})."
319 - )
320 -
321 - kernel_type = self.kernel_type
322 - # Validate output_size_per_partition
323 - output_size_per_partition = sum(output_partition_sizes)
324 -
325 - is_row_parallel = input_size != input_size_per_partition
326 - weight_loader = extra_weight_attrs.get("weight_loader")
327 -
328 - mp_linear_kernel_config = MPLinearLayerConfig(
329 - full_weight_shape=(input_size, output_size),
330 - partition_weight_shape=(
331 - input_size_per_partition,
332 - output_size_per_partition,
333 - ),
334 - weight_type=self.quant_config.quant_type,
335 - act_type=params_dtype,
336 - group_size=self.quant_config.group_size,
337 - zero_points=False,
338 - has_g_idx=self.quant_config.desc_act,
339 - )
340 -
341 - if kernel_type.__name__ not in self._kernel_backends_being_used:
342 - logger.info("Using %s for GPTQBitBLASLinearMethod", kernel_type.__name__)
343 - self._kernel_backends_being_used.add(kernel_type.__name__)
344 -
345 - # Normalize group_size
346 - if self.quant_config.group_size != -1:
347 - group_size = self.quant_config.group_size
348 - else:
349 - group_size = input_size
350 -
351 - # Determine sharding
352 - if bitblas_repeat_scales_on_all_ranks(
353 - self.quant_config.desc_act, self.quant_config.group_size, is_row_parallel
354 - ):
355 - # By setting scale_dim == None, weight_loader will
356 - # repeat the scales on each GPU in TP>1 case.
357 - scales_and_zp_input_dim = None
358 - scales_and_zp_size = input_size // group_size
359 - else:
360 - # By setting scale_dim == 0, weight_loader will
361 - # shard the scales in TP>1 case.
362 - scales_and_zp_input_dim = 0
363 - scales_and_zp_size = input_size_per_partition // group_size
364 -
365 - # Init buffers
366 - # Quantized weights
367 - qweight = PackedvLLMParameter(
368 - data=torch.empty(
369 - input_size_per_partition // self.quant_config.pack_factor,
370 - output_size_per_partition,
371 - dtype=torch.int32,
372 - ),
373 - input_dim=0,
374 - output_dim=1,
375 - packed_dim=0,
376 - packed_factor=self.quant_config.pack_factor,
377 - weight_loader=weight_loader,
378 - )
379 -
380 - # Activation order
381 - # Ignore warning from fused linear layers such as QKVParallelLinear.
382 - g_idx = RowvLLMParameter(
383 - data=torch.empty(
384 - input_size_per_partition,
385 - dtype=torch.int32,
386 - ),
387 - input_dim=0,
388 - weight_loader=weight_loader,
389 - )
390 -
391 - # Scales
392 - scales = Parameter(
393 - torch.empty(
394 - scales_and_zp_size,
395 - output_size_per_partition,
396 - dtype=params_dtype,
397 - ),
398 - requires_grad=False,
399 - )
400 - set_weight_attrs(
401 - scales,
402 - {
403 - **extra_weight_attrs,
404 - "input_dim": scales_and_zp_input_dim,
405 - "output_dim": 1,
406 - },
407 - )
408 -
409 - # Quantized zero-points
410 - qzeros_args = {
411 - "data": torch.empty(
412 - scales_and_zp_size,
413 - output_size_per_partition // self.quant_config.pack_factor,
414 - dtype=torch.int32,
415 - ),
416 - "weight_loader": weight_loader,
417 - }
418 - weight_scale_args = {
419 - "data": torch.empty(
420 - scales_and_zp_size,
421 - output_size_per_partition,
422 - dtype=params_dtype,
423 - ),
424 - "weight_loader": weight_loader,
425 - }
426 -
427 - if scales_and_zp_input_dim is None:
428 - scales = ChannelQuantScaleParameter(output_dim=1, **weight_scale_args)
429 - qzeros = PackedColumnParameter(
430 - output_dim=1,
431 - packed_dim=1,
432 - packed_factor=self.quant_config.pack_factor,
433 - **qzeros_args,
434 - )
435 -
436 - else:
437 - scales = GroupQuantScaleParameter(
438 - output_dim=1, input_dim=0, **weight_scale_args
439 - )
440 - qzeros = PackedvLLMParameter(
441 - input_dim=0,
442 - output_dim=1,
443 - packed_dim=1,
444 - packed_factor=self.quant_config.pack_factor,
445 - **qzeros_args,
446 - )
447 -
448 - layer.register_parameter("qweight", qweight)
449 - layer.register_parameter("g_idx", g_idx)
450 - layer.register_parameter("scales", scales)
451 - layer.register_parameter("qzeros", qzeros)
452 -
453 - self.kernel = kernel_type(
454 - mp_linear_kernel_config,
455 - w_q_param_name="qweight",
456 - w_s_param_name="scales",
457 - w_zp_param_name="qzeros",
458 - w_gidx_param_name="g_idx",
459 - bitblas_quant_config=self.quant_config,
460 - )
461 -
462 - # Initialize or retrieve the BitBLAS matrix multiplication operator.
463 - self.kernel.configure_bitblas_matmul(
464 - input_size_per_partition,
465 - output_size_per_partition,
466 - params_dtype=params_dtype,
467 - bias=False,
468 - )
469 -
470 - def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
471 - self.kernel.process_weights_after_loading(layer)
472 -
473 - def apply(
474 - self,
475 - layer: torch.nn.Module,
476 - x: torch.Tensor,
477 - bias: torch.Tensor | None = None,
478 - ) -> torch.Tensor:
479 - out = self.kernel.apply_gptq_bitblas_linear(layer, x)
480 - if bias is not None:
481 - out.add_(bias)
482 - return out
vllm/model_executor/layers/quantization/kernels/mixed_precision/__init__.py
-4
@@ -5,9 +5,6 @@ import vllm.envs as envs
5 from vllm.model_executor.layers.quantization.kernels.mixed_precision.allspark import ( # noqa: E501
6 AllSparkLinearKernel,
7 )
8 -from vllm.model_executor.layers.quantization.kernels.mixed_precision.bitblas import ( # noqa: E501
9 - BitBLASLinearKernel,
10 -)
8 from vllm.model_executor.layers.quantization.kernels.mixed_precision.conch import ( # noqa: E501
9 ConchLinearKernel,
10 )
@@ -45,7 +42,6 @@ _POSSIBLE_KERNELS: list[type[MPLinearKernel]] = [
42 AllSparkLinearKernel,
43 MarlinLinearKernel,
44 Dynamic4bitLinearKernel,
48 - BitBLASLinearKernel,
45 ConchLinearKernel,
46 ExllamaLinearKernel,
47 XPUwNa16LinearKernel,
vllm/model_executor/layers/quantization/kernels/mixed_precision/bitblas.py deleted
-323
@@ -1,323 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -
5 -import torch
6 -from packaging import version
7 -
8 -from vllm.logger import init_logger
9 -from vllm.model_executor.layers.quantization import QuantizationConfig
10 -from vllm.model_executor.layers.quantization.utils import replace_parameter
11 -from vllm.model_executor.layers.quantization.utils.bitblas_utils import (
12 - BITBLAS_OPTIMIZE_FEATURES,
13 - BITBLAS_SUPPORTED_GROUP_SIZES,
14 - MINIMUM_BITBLAS_VERSION,
15 - bitblas_make_empty_g_idx,
16 - bitblas_sort_g_idx,
17 - check_bitblas_supports_shape,
18 - query_bitblas_supported_quant_types,
19 - unpack_gptq_qweight,
20 - unpack_gptq_qzeros,
21 -)
22 -
23 -from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
24 -
25 -logger = init_logger(__name__)
26 -
27 -
28 -class BitBLASLinearKernel(MPLinearKernel):
29 - OPT_FEATURES: list[int] = BITBLAS_OPTIMIZE_FEATURES
30 - ENABLE_TUNING: bool = True
31 - MATMUL_LAYOUT: str = "nt"
32 - BITBLAS_DTYPES: dict[torch.dtype, str] = {
33 - torch.float32: "float32",
34 - torch.float16: "float16",
35 - torch.bfloat16: "bfloat16",
36 - torch.half: "float16",
37 - torch.int8: "int8",
38 - }
39 - bitblas_matmul: object = None
40 -
41 - def __init__(
42 - self,
43 - c: MPLinearLayerConfig,
44 - w_q_param_name: str,
45 - w_s_param_name: str,
46 - w_zp_param_name: str | None = None,
47 - w_gidx_param_name: str | None = None,
48 - bitblas_quant_config: QuantizationConfig | None = None,
49 - ):
50 - self.quant_config = bitblas_quant_config
51 - super().__init__(
52 - c, w_q_param_name, w_s_param_name, w_zp_param_name, w_gidx_param_name
53 - )
54 -
55 - def repack_bitblas_from_gptq(
56 - self,
57 - b_q_weight: torch.Tensor,
58 - scales: torch.Tensor,
59 - qzeros: torch.Tensor | None = None,
60 - ):
61 - from bitblas.quantization.utils import general_compress
62 -
63 - assert self.bitblas_matmul is not None, "bitblas_matmul is None"
64 -
65 - quant_config = self.quant_config
66 - # qweight in gptq old quant linear stored with
67 - # (outfeatures, infeatures), should be transposed.
68 - qweight = b_q_weight.T.contiguous().view(quant_config.torch_storage_dtype) # type: ignore[union-attr]
69 - intweight = unpack_gptq_qweight(qweight, quant_config.weight_bits).contiguous() # type: ignore[union-attr]
70 - if self.bitblas_matmul.weight_transform is not None: # type: ignore[attr-defined]
71 - qweight = self.bitblas_matmul.weight_transform( # type: ignore[attr-defined]
72 - intweight.cpu()
73 - ).cuda()
74 - # scales in gptq old quant linear stored with
75 - # (infeatures // group_size, outfeatures), should be transposed.
76 - scales = scales.T.contiguous()
77 -
78 - if qzeros is None:
79 - return qweight, scales, None
80 -
81 - # qzeros should be de-quantized to int zeros.
82 - weight_bits = quant_config.weight_bits # type: ignore[union-attr]
83 - intzeros = unpack_gptq_qzeros(qzeros, weight_bits).T.contiguous()
84 - zeros: torch.Tensor | None = None
85 - zeros_mode = self.bitblas_matmul.config.zeros_mode # type: ignore[attr-defined]
86 - if zeros_mode == "original":
87 - zeros = intzeros.to(torch.float16).contiguous()
88 - elif zeros_mode == "rescale":
89 - assert zeros is not None, "zeros should not be None"
90 - zeros[:, :] = intzeros.to(torch.float16)[:, :] * scales[:, :]
91 - elif zeros_mode == "quantized":
92 - zeros = (
93 - torch.Tensor(
94 - general_compress(
95 - intzeros.T.contiguous().cpu().numpy(),
96 - weight_bits,
97 - )
98 - )
99 - .to(qweight.device)
100 - .to(
101 - quant_config.torch_storage_dtype # type: ignore[union-attr]
102 - )
103 - .contiguous()
104 - )
105 - else:
106 - raise ValueError("Unsupported zeros type: {}".format(zeros_mode))
107 -
108 - return qweight, scales, zeros
109 -
110 - @classmethod
111 - def get_min_capability(cls) -> int:
112 - return 70
113 -
114 - @classmethod
115 - def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
116 - is_bitblas_installed = True
117 -
118 - try:
119 - import bitblas
120 -
121 - if version.parse(bitblas.__version__) < version.parse(
122 - MINIMUM_BITBLAS_VERSION
123 - ):
124 - raise ImportError(
125 - "bitblas version is wrong. Please "
126 - f"install bitblas>={MINIMUM_BITBLAS_VERSION}"
127 - )
128 - except ImportError:
129 - is_bitblas_installed = False
130 -
131 - if not is_bitblas_installed:
132 - return (
133 - False,
134 - "bitblas is not installed. Please install bitblas "
135 - "by running `pip install bitblas>="
136 - f"{MINIMUM_BITBLAS_VERSION}`",
137 - )
138 -
139 - quant_types = query_bitblas_supported_quant_types(c.zero_points)
140 - if c.weight_type not in quant_types:
141 - return False, (
142 - f"Quant type ({c.weight_type}) not supported by"
143 - f" BitBLAS, supported types are: {quant_types}"
144 - )
145 -
146 - if c.group_size not in BITBLAS_SUPPORTED_GROUP_SIZES:
147 - return False, (
148 - f"Group size ({c.group_size}) not supported by "
149 - "BitBLAS, supported group sizes are: "
150 - f"{BITBLAS_SUPPORTED_GROUP_SIZES}"
151 - )
152 -
153 - return check_bitblas_supports_shape(
154 - c.partition_weight_shape[1], # out_features
155 - c.partition_weight_shape[0], # in_features
156 - c.full_weight_shape[0], # in_features
157 - c.group_size,
158 - )
159 -
160 - # note assumes that
161 - # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}
162 - # `weight_scale` is: {input_dim = 0, output_dim = 1}
163 - def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
164 - device = getattr(layer, self.w_q_name).device
165 - c = self.config
166 - quant_config = self.quant_config
167 -
168 - # Default names since bitblas requires empty parameters for these,
169 - # TODO: remove this requirement from bitblas (allow optional tensors)
170 - if getattr(self, "w_gidx_name", None) is None:
171 - self.w_gidx_name: str = "g_idx"
172 - if getattr(self, "w_zp_name", None) is None:
173 - self.w_zp_name: str = "qzeros"
174 -
175 - if c.has_g_idx:
176 - g_idx, g_idx_sort_indices = bitblas_sort_g_idx(
177 - getattr(layer, self.w_gidx_name)
178 - )
179 - self._transform_param(layer, self.w_gidx_name, lambda _: g_idx)
180 - layer.g_idx_sort_indices = g_idx_sort_indices
181 - else:
182 - setattr(layer, self.w_gidx_name, bitblas_make_empty_g_idx(device))
183 - layer.g_idx_sort_indices = bitblas_make_empty_g_idx(device)
184 -
185 - if c.zero_points:
186 - raise NotImplementedError("Zero points not supported by BitBLAS")
187 - else:
188 - setattr(layer, self.w_zp_name, bitblas_make_empty_g_idx(device))
189 -
190 - # Repack weights
191 - bitblas_qweight, bitblas_scales, bitblas_qzeros = self.repack_bitblas_from_gptq(
192 - layer.qweight,
193 - layer.scales,
194 - None if quant_config.is_sym else layer.qzeros, # type: ignore[union-attr]
195 - )
196 - replace_parameter(layer, self.w_q_name, bitblas_qweight)
197 - replace_parameter(layer, self.w_s_name, bitblas_scales)
198 - if bitblas_qzeros is not None:
199 - replace_parameter(layer, self.w_zp_name, bitblas_qzeros)
200 -
201 - def configure_bitblas_matmul(
202 - self,
203 - infeatures: int,
204 - outfeatures: int,
205 - params_dtype: torch.dtype,
206 - bias: bool,
207 - ) -> None:
208 - enable_tuning = self.ENABLE_TUNING
209 - layout = self.MATMUL_LAYOUT
210 - bits = self.quant_config.weight_bits # type: ignore[union-attr]
211 - self._configure_bitblas_matmul(
212 - infeatures,
213 - outfeatures,
214 - params_dtype,
215 - enable_tuning,
216 - bias,
217 - layout,
218 - bits,
219 - )
220 -
221 - def _configure_bitblas_matmul(
222 - self,
223 - infeatures,
224 - outfeatures,
225 - params_dtype,
226 - enable_tuning,
227 - bias,
228 - layout,
229 - bits,
230 - ):
231 - from bitblas import MatmulConfig
232 -
233 - bitblas_dtype = self.BITBLAS_DTYPES[params_dtype]
234 - quant_config = self.quant_config
235 - with_scaling = False
236 - with_zeros = False
237 - group_size = quant_config.group_size # type: ignore[union-attr]
238 - zeros_mode = quant_config.zeros_mode # type: ignore[union-attr]
239 - if quant_config.quant_method == "gptq": # type: ignore[union-attr]
240 - with_scaling = True
241 - with_zeros = True
242 - W_dtype = f"uint{bits}"
243 - if quant_config.is_sym: # type: ignore[union-attr]
244 - with_zeros = False
245 - W_dtype = f"int{bits}"
246 - else:
247 - raise ValueError(
248 - f"Unsupported quant_method {quant_config.quant_method}" # type: ignore[union-attr]
249 - ) # type: ignore[union-attr]
250 -
251 - matmul_config = MatmulConfig(
252 - M=self.OPT_FEATURES,
253 - N=outfeatures,
254 - K=infeatures,
255 - A_dtype=bitblas_dtype,
256 - W_dtype=W_dtype,
257 - out_dtype=bitblas_dtype,
258 - accum_dtype="int32" if bitblas_dtype == "int8" else bitblas_dtype,
259 - storage_dtype=quant_config. # type: ignore[union-attr]
260 - storage_dtype, # type: ignore[union-attr]
261 - with_scaling=with_scaling,
262 - with_zeros=with_zeros,
263 - group_size=group_size,
264 - with_bias=bias,
265 - layout=layout,
266 - zeros_mode=zeros_mode,
267 - )
268 - self.bitblas_matmul = self._get_or_create_bitblas_operator(
269 - matmul_config, enable_tuning
270 - )
271 -
272 - def _get_or_create_bitblas_operator(self, config, enable_tuning):
273 - from bitblas import Matmul, auto_detect_nvidia_target
274 - from bitblas.cache import get_database_path, global_operator_cache
275 -
276 - BITBLAS_DATABASE_PATH = get_database_path()
277 - BITBLAS_TARGET = auto_detect_nvidia_target()
278 -
279 - if global_operator_cache.size() == 0:
280 - global_operator_cache.load_from_database(
281 - BITBLAS_DATABASE_PATH, BITBLAS_TARGET
282 - )
283 -
284 - bitblas_matmul = global_operator_cache.get(config)
285 - if bitblas_matmul is None:
286 - bitblas_matmul = Matmul(config, target=BITBLAS_TARGET, enable_tuning=False)
287 - if enable_tuning:
288 - bitblas_matmul.hardware_aware_finetune(topk=20)
289 - global_operator_cache.add(config, bitblas_matmul)
290 - global_operator_cache.save_into_database(
291 - BITBLAS_DATABASE_PATH, BITBLAS_TARGET
292 - )
293 - TUNING_MESSAGE = (
294 - f"BitBLAS Operator {config} tuned and saved to database."
295 - )
296 - logger.info(TUNING_MESSAGE)
297 - else:
298 - _message = f"BitBLAS Operator {config} created without tuning. "
299 - logger.info(_message)
300 - else:
301 - _message = f"BitBLAS Operator {config} retrieved from cache."
302 - logger.info(_message)
303 - return bitblas_matmul
304 -
305 - def apply_gptq_bitblas_linear(
306 - self,
307 - layer: torch.nn.Module,
308 - x: torch.Tensor,
309 - ) -> torch.Tensor:
310 - output_size_per_partition = self.config.partition_weight_shape[1]
311 - out_shape = x.shape[:-1] + (output_size_per_partition,)
312 - args = [x, layer.qweight, layer.scales]
313 - if self.bitblas_matmul.config.with_zeros: # type: ignore[attr-defined]
314 - args.append(layer.qzeros)
315 - output = self.bitblas_matmul(*args) # type: ignore[operator]
316 - return output.view(out_shape)
317 -
318 - def apply_weights(self, layer, x, bias=None):
319 - NOT_IMPLEMENT_MESSAGE = (
320 - f"{self.__class__.__name__}.apply_weights is not implemented. "
321 - "Please use BitBLASLinearKernel.apply_gptq_bitblas_linear instead"
322 - )
323 - raise NotImplementedError(NOT_IMPLEMENT_MESSAGE)
vllm/model_executor/layers/quantization/utils/bitblas_utils.py deleted
-229
@@ -1,229 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import torch
5 -from packaging import version
6 -
7 -from vllm.platforms import current_platform
8 -from vllm.scalar_type import ScalarType, scalar_types
9 -
10 -MINIMUM_BITBLAS_VERSION = "0.1.0"
11 -
12 -BITBLAS_MIN_WEIGHT_SIZE_N = 16
13 -BITBLAS_MIN_WEIGHT_SIZE_K = 16
14 -GPTQ_BITBLAS_MAX_PARALLEL = 16
15 -
16 -BITBLAS_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
17 -
18 -# For dynamic shape code generation
19 -BITBLAS_OPTIMIZE_FEATURES = [1, 16, 32, 64, 128, 256, 512, 1024]
20 -# If want to enable high performance for contiguous batching
21 -# Please use the following values
22 -BITBLAS_OPTIMIZE_FEATURES_CONTIGUOUS = [16, 32, 64, 128, 256, 512, 1024]
23 -
24 -BITBLAS_SUPPORTED_NUM_BITS = [1, 2, 4, 8]
25 -BITBLAS_SUPPORTED_SYM = [False, True]
26 -
27 -
28 -# Determines the supported quantization types for BitBLAS based on the
29 -# device's capability and whether zero-point (zp) is used.
30 -def query_bitblas_supported_quant_types(
31 - has_zp: bool, device_capability: int | None = None
32 -):
33 - if device_capability is None:
34 - capability_tuple = current_platform.get_device_capability()
35 - device_capability = (
36 - -1 if capability_tuple is None else capability_tuple.to_int()
37 - )
38 -
39 - if device_capability < 70:
40 - return []
41 -
42 - if has_zp:
43 - # AWQ style, unsigned + runtime zero-point
44 - return [scalar_types.uint4, scalar_types.uint8]
45 - else:
46 - # GPTQ style, unsigned + symmetric bias
47 - # TODO: once fp8_bitblas is merged into "gptq_bitblas" we should be able
48 - # to add `scalar_types.float8_e4m3fn` here
49 - return [scalar_types.uint4b8, scalar_types.uint8b128]
50 -
51 -
52 -def _check_bitblas_supported(
53 - quant_type: ScalarType,
54 - group_size: int | None,
55 - has_zp: bool,
56 - device_capability: int | None = None,
57 -) -> tuple[bool, str | None]:
58 - if device_capability is None:
59 - capability_tuple = current_platform.get_device_capability()
60 - device_capability = (
61 - -1 if capability_tuple is None else capability_tuple.to_int()
62 - )
63 -
64 - supported_types = query_bitblas_supported_quant_types(has_zp, device_capability)
65 -
66 - if quant_type not in supported_types:
67 - return (
68 - False,
69 - f"BitBLAS does not support weight_bits = {quant_type}. "
70 - f"Only types = {supported_types} "
71 - f"are supported (for group_size = {group_size}, "
72 - f"device_capability = {device_capability}, zp = {has_zp}).",
73 - )
74 - if group_size is None or group_size not in BITBLAS_SUPPORTED_GROUP_SIZES:
75 - return (
76 - False,
77 - f"BitBLAS does not support group_size = {group_size}. "
78 - f"Only group_sizes = {BITBLAS_SUPPORTED_GROUP_SIZES} "
79 - "are supported.",
80 - )
81 -
82 - # Finally, check if bitblas is installed
83 - try:
84 - import bitblas
85 -
86 - if version.parse(bitblas.__version__) < version.parse(MINIMUM_BITBLAS_VERSION):
87 - raise ImportError(
88 - "bitblas version is wrong. Please "
89 - f"install bitblas>={MINIMUM_BITBLAS_VERSION}"
90 - )
91 - except ImportError:
92 - return False, "BitBLAS is not installed."
93 -
94 - return True, None
95 -
96 -
97 -def check_bitblas_supported(
98 - quant_type: ScalarType,
99 - group_size: int,
100 - has_zp: bool = False,
101 - device_capability: int | None = None,
102 -) -> bool:
103 - cond, _ = _check_bitblas_supported(
104 - quant_type, group_size, has_zp, device_capability
105 - )
106 - return cond
107 -
108 -
109 -def verify_bitblas_supported(
110 - quant_type: ScalarType, group_size: int, has_zp: bool = False
111 -) -> None:
112 - cond, err_msg = _check_bitblas_supported(quant_type, group_size, has_zp)
113 - if not cond:
114 - assert err_msg is not None
115 - raise ValueError(err_msg)
116 -
117 -
118 -def verify_bitblas_supports_shape(
119 - output_size_per_partition: int,
120 - input_size_per_partition: int,
121 - input_size: int,
122 - group_size: int,
123 -) -> None:
124 - # Validate output_size_per_partition
125 - if output_size_per_partition % BITBLAS_MIN_WEIGHT_SIZE_N != 0:
126 - raise ValueError(
127 - f"Weight output_size_per_partition = "
128 - f"{output_size_per_partition} is not divisible by "
129 - f" min_thread_n = {BITBLAS_MIN_WEIGHT_SIZE_N}. "
130 - "Consider reducing tensor_parallel_size or running "
131 - "with --quantization gptq."
132 - )
133 -
134 - # Validate input_size_per_partition
135 - if input_size_per_partition % BITBLAS_MIN_WEIGHT_SIZE_K != 0:
136 - raise ValueError(
137 - f"Weight input_size_per_partition = "
138 - f"{input_size_per_partition} is not divisible "
139 - f"by min_thread_k = {BITBLAS_MIN_WEIGHT_SIZE_K}. "
140 - "Consider reducing tensor_parallel_size or running "
141 - "with --quantization gptq."
142 - )
143 -
144 - if group_size < input_size and input_size_per_partition % group_size != 0:
145 - raise ValueError(
146 - f"Weight input_size_per_partition = {input_size_per_partition}"
147 - f" is not divisible by group_size = {group_size}."
148 - "Consider reducing tensor_parallel_size or running "
149 - "with --quantization gptq."
150 - )
151 -
152 -
153 -def check_bitblas_supports_shape(
154 - output_size_per_partition: int,
155 - input_size_per_partition: int,
156 - input_size: int,
157 - group_size: int,
158 -) -> tuple[bool, str | None]:
159 - try:
160 - verify_bitblas_supports_shape(
161 - output_size_per_partition, input_size_per_partition, input_size, group_size
162 - )
163 - except ValueError as e:
164 - return False, e.__str__()
165 - return True, None
166 -
167 -
168 -def bitblas_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:
169 - return (not act_order) or (act_order and not is_row_parallel)
170 -
171 -
172 -def bitblas_repeat_scales_on_all_ranks(
173 - act_order: bool, group_size: int, is_row_parallel: bool
174 -) -> bool:
175 - # Need to repeat scales on every rank if act_ordering or
176 - # channelwise and RowParallelLinear
177 - is_channelwise = group_size == -1
178 - return act_order or (is_channelwise and is_row_parallel)
179 -
180 -
181 -def bitblas_make_empty_g_idx(device: torch.device) -> torch.Tensor:
182 - return torch.nn.Parameter(
183 - torch.empty(0, dtype=torch.int, device=device), requires_grad=False
184 - )
185 -
186 -
187 -def bitblas_make_empty_zp(device: torch.device) -> torch.Tensor:
188 - return torch.nn.Parameter(
189 - torch.empty(0, dtype=torch.int, device=device), requires_grad=False
190 - )
191 -
192 -
193 -def bitblas_sort_g_idx(g_idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
194 - g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)
195 - return g_idx[g_idx_sort_indices], g_idx_sort_indices
196 -
197 -
198 -def unpack_gptq_qzeros(qzeros, bits, is_gptq_v2=False) -> torch.Tensor:
199 - qzeros = qzeros.view(torch.int32)
200 - elems_per_int32 = 32 // bits
201 - unpacked_zeros = torch.zeros(
202 - (qzeros.shape[0], qzeros.shape[1] * elems_per_int32),
203 - dtype=torch.int8,
204 - device=qzeros.device,
205 - requires_grad=False,
206 - )
207 -
208 - for col in range(unpacked_zeros.shape[1]):
209 - i = col % elems_per_int32
210 - unpacked_zeros[:, col] = (qzeros[:, col // elems_per_int32] >> (bits * i)) & 0xF
211 - if not is_gptq_v2:
212 - return unpacked_zeros + 1
213 - return unpacked_zeros
214 -
215 -
216 -def unpack_gptq_qweight(qweight, bits):
217 - qweight = qweight.view(torch.int8)
218 - elems_per_int8 = 8 // bits
219 - unpacked_weight = torch.zeros(
220 - (qweight.shape[0], qweight.shape[1] * elems_per_int8),
221 - dtype=torch.int8,
222 - device=qweight.device,
223 - requires_grad=False,
224 - )
225 - for col in range(unpacked_weight.shape[1]):
226 - i = col % elems_per_int8
227 - unpacked_weight[:, col] = qweight[:, col // elems_per_int8] >> (bits * i)
228 -
229 - return torch.bitwise_and(unpacked_weight, 2**bits - 1)
vllm/model_executor/parameter.py
+1 -25
@@ -322,13 +322,11 @@ class PackedColumnParameter(_ColumnvLLMParameter):
322 packed_factor: int | Fraction,
323 packed_dim: int,
324 marlin_tile_size: int | None = None,
325 - bitblas_tile_size: int | None = None,
325 **kwargs,
326 ):
327 self._packed_factor = packed_factor
328 self._packed_dim = packed_dim
329 self._marlin_tile_size = marlin_tile_size
331 - self._bitblas_tile_size = bitblas_tile_size
330 super().__init__(**kwargs)
331
332 @property
@@ -343,17 +341,12 @@ class PackedColumnParameter(_ColumnvLLMParameter):
341 def marlin_tile_size(self):
342 return self._marlin_tile_size
343
346 - @property
347 - def bitblas_tile_size(self):
348 - return self._bitblas_tile_size
349 -
344 def adjust_shard_indexes_for_packing(self, shard_size, shard_offset):
345 return _adjust_shard_indexes_for_packing(
346 shard_size=shard_size,
347 shard_offset=shard_offset,
348 packed_factor=self.packed_factor,
349 marlin_tile_size=self.marlin_tile_size,
356 - bitblas_tile_size=self.bitblas_tile_size,
350 )
351
352
@@ -373,13 +366,11 @@ class PackedvLLMParameter(ModelWeightParameter):
366 packed_factor: int | Fraction,
367 packed_dim: int,
368 marlin_tile_size: int | None = None,
376 - bitblas_tile_size: int | None = None,
369 **kwargs,
370 ):
371 self._packed_factor = packed_factor
372 self._packed_dim = packed_dim
373 self._marlin_tile_size = marlin_tile_size
382 - self._bitblas_tile_size = bitblas_tile_size
374 super().__init__(**kwargs)
375
376 @property
@@ -394,17 +385,12 @@ class PackedvLLMParameter(ModelWeightParameter):
385 def marlin_tile_size(self):
386 return self._marlin_tile_size
387
397 - @property
398 - def bitblas_tile_size(self):
399 - return self._bitblas_tile_size
400 -
388 def adjust_shard_indexes_for_packing(self, shard_size, shard_offset):
389 return _adjust_shard_indexes_for_packing(
390 shard_size=shard_size,
391 shard_offset=shard_offset,
392 packed_factor=self.packed_factor,
393 marlin_tile_size=self.marlin_tile_size,
407 - bitblas_tile_size=self.bitblas_tile_size,
394 )
395
396
@@ -617,12 +603,8 @@ def _adjust_shard_indexes_for_marlin(shard_size, shard_offset, marlin_tile_size)
603 return shard_size * marlin_tile_size, shard_offset * marlin_tile_size
604
605
620 -def _adjust_shard_indexes_for_bitblas(shard_size, shard_offset, bitblas_tile_size):
621 - return shard_size // bitblas_tile_size, shard_offset // bitblas_tile_size
622 -
623 -
606 def _adjust_shard_indexes_for_packing(
625 - shard_size, shard_offset, packed_factor, marlin_tile_size, bitblas_tile_size
607 + shard_size, shard_offset, packed_factor, marlin_tile_size
608 ):
609 shard_size = shard_size // packed_factor
610 shard_offset = shard_offset // packed_factor
@@ -632,11 +614,5 @@ def _adjust_shard_indexes_for_packing(
614 shard_offset=shard_offset,
615 marlin_tile_size=marlin_tile_size,
616 )
635 - elif bitblas_tile_size is not None:
636 - return _adjust_shard_indexes_for_bitblas(
637 - shard_size=shard_size,
638 - shard_offset=shard_offset,
639 - bitblas_tile_size=bitblas_tile_size,
640 - )
617
618 return shard_size, shard_offset