A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[ Misc ] Refactor Marlin Python Utilities (#6082)
Co-authored-by: Robert Shaw <rshaw@neuralmagic.com>
Robert Shaw committed
Jul 11, 2024 at 11:40 UTC
b675069d7486129dbed7847f420b7a927691f16b
12 files changed
+704
-742
benchmarks/kernels/benchmark_marlin.py
+6
-4
@@ -5,14 +5,16 @@ import torch.utils.benchmark as benchmark
5
from benchmark_shapes import WEIGHT_SHAPES
6
7
from vllm import _custom_ops as ops
8
-from vllm.model_executor.layers.quantization.gptq_marlin import (
9
- GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
10
- GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
8
from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
9
GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
10
GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
11
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15
- MarlinWorkspace, marlin_24_quantize, marlin_quantize)
12
+ GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
13
+ GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
14
+from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
15
+ MarlinWorkspace, marlin_quantize)
16
+from vllm.model_executor.layers.quantization.utils.marlin_utils_test_24 import (
17
+ marlin_24_quantize)
18
from vllm.model_executor.layers.quantization.utils.quant_utils import (
19
gptq_pack, quantize_weights, sort_weights)
20
from vllm.utils import FlexibleArgumentParser
tests/kernels/test_marlin_gemm.py
+25
-21
@@ -5,19 +5,21 @@ Run `pytest tests/kernels/marlin/test_marlin_gemm.py`.
5
import pytest
6
import torch
7
8
+from tests.quantization.utils import is_quant_method_supported
9
from vllm import _custom_ops as ops
9
-from vllm.model_executor.layers.quantization.gptq_marlin import (
10
- GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
11
- GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS,
12
- marlin_permute_scales)
10
from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
11
GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
12
GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
16
-from vllm.model_executor.layers.quantization.utils.marlin_perms import (
17
- marlin_perm)
13
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
19
- MarlinWorkspace, compute_max_diff, is_marlin_supported, marlin_24_quantize,
20
- marlin_quantize, marlin_weights, pack_fp8_to_int32)
14
+ GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
15
+ GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS,
16
+ marlin_permute_scales)
17
+from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
18
+ pack_fp8_to_int32)
19
+from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
20
+ MarlinWorkspace, get_weight_perm, marlin_quantize, marlin_weights)
21
+from vllm.model_executor.layers.quantization.utils.marlin_utils_test_24 import (
22
+ marlin_24_quantize)
23
from vllm.model_executor.layers.quantization.utils.quant_utils import (
24
gptq_pack, quantize_weights, sort_weights)
25
@@ -42,11 +44,16 @@ MNK_FACTORS = [
44
DTYPES = [torch.float16, torch.bfloat16]
45
46
47
+def compute_max_diff(output, output_ref):
48
+ return torch.mean(torch.abs(output - output_ref)) / torch.mean(
49
+ torch.abs(output_ref))
50
+
51
+
52
def rand_data(shape, dtype=torch.float16):
53
return torch.randn(shape, dtype=dtype, device="cuda")
54
55
49
-@pytest.mark.skipif(not is_marlin_supported(),
56
+@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
57
reason="Marlin is not supported on this GPU type.")
58
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
59
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
@@ -93,8 +100,8 @@ def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
100
q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
101
102
# Pack to Marlin format
96
- marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits,
97
- marlin_perm[num_bits])
103
+ weight_perm = get_weight_perm(num_bits)
104
+ marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
105
106
# Run Marlin repack GPU kernel
107
marlin_q_w_2 = ops.gptq_marlin_repack(
@@ -109,7 +116,7 @@ def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
116
assert torch.allclose(marlin_q_w_1, marlin_q_w_2)
117
118
112
-@pytest.mark.skipif(not is_marlin_supported(),
119
+@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
120
reason="Marlin is not supported on this GPU type.")
121
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
122
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
@@ -174,7 +181,7 @@ def test_marlin_gemm(
181
assert max_diff < 0.04
182
183
177
-@pytest.mark.skipif(not is_marlin_supported(),
184
+@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
185
reason="Marlin is not supported on this GPU type.")
186
@pytest.mark.parametrize("k_chunk", MARLIN_24_K_CHUNKS)
187
@pytest.mark.parametrize("n_chunk", MARLIN_24_N_CHUNKS)
@@ -222,7 +229,7 @@ def test_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size, mnk_factors):
229
assert max_diff < 0.04
230
231
225
-@pytest.mark.skipif(not is_marlin_supported(),
232
+@pytest.mark.skipif(not is_quant_method_supported("fp8"),
233
reason="Marlin is not supported on this GPU type.")
234
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
235
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
@@ -268,13 +275,10 @@ def test_fp8_marlin_gemm(
275
# expand it to channelwise
276
scales = weight_scale.repeat(1, size_n).to(a_input.dtype).to("cuda")
277
# Permute scales
271
- marlin_scales = marlin_permute_scales(
272
- s=scales,
273
- size_k=size_k,
274
- size_n=size_n,
275
- group_size=-1,
276
- num_bits=8,
277
- )
278
+ marlin_scales = marlin_permute_scales(s=scales,
279
+ size_k=size_k,
280
+ size_n=size_n,
281
+ group_size=-1)
282
283
workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
284
GPTQ_MARLIN_MAX_PARALLEL)
tests/quantization/test_compressed_tensors.py
+14
-9
@@ -6,7 +6,6 @@ Run `pytest tests/quantization/test_compressed_tensors.py`.
6
import pytest
7
import torch
8
9
-from vllm import SamplingParams
9
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
10
CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
11
CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
@@ -57,12 +56,14 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
56
assert qkv_proj.weight_scale.dtype is torch.float32
57
assert qkv_proj.input_scale.dtype is torch.float32
58
59
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
60
+ assert output
61
+
62
63
def test_compressed_tensors_no_enforce_eager(vllm_runner):
64
model_path = "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change"
65
with vllm_runner(model_path) as llm:
64
- sampling_params = SamplingParams()
65
- output = llm.generate("Hello world!", sampling_params=sampling_params)
66
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
67
assert output
68
69
@@ -84,13 +85,16 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
85
assert qkv_proj.scheme.strategy == strategy
86
assert qkv_proj.weight.dtype is torch.int8
87
88
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
89
+ assert output
90
+
91
92
@pytest.mark.parametrize(
93
"wNa16_args",
94
[("nm-testing/tinyllama-oneshot-w4a16-channel-v2", "channel", None, 8),
95
("nm-testing/tinyllama-oneshot-w4a16-group128-v2", "group", 128, 8),
96
("nm-testing/tinyllama-oneshot-w8a16-per-channel", "channel", None, 4)])
93
-def test_compressed_tensors_w4a16(vllm_runner, wNa16_args):
97
+def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
98
model, strategy, group, pack_factor = wNa16_args
99
with vllm_runner(model) as llm:
100
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
@@ -101,12 +105,15 @@ def test_compressed_tensors_w4a16(vllm_runner, wNa16_args):
105
assert isinstance(qkv_proj.scheme, CompressedTensorsWNA16)
106
107
assert qkv_proj.scheme.strategy == strategy
104
- assert qkv_proj.scheme.group_size == group
108
+ assert qkv_proj.scheme.group_size == (-1 if group is None else group)
109
110
assert qkv_proj.weight_packed.dtype is torch.int32
111
assert qkv_proj.weight_scale.dtype is torch.float16
112
assert qkv_proj.weight_packed.pack_factor == pack_factor
113
114
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
115
+ assert output
116
+
117
118
def test_compressed_tensors_w4a16_marlin24(vllm_runner):
119
model_path = "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t"
@@ -120,8 +127,7 @@ def test_compressed_tensors_w4a16_marlin24(vllm_runner):
127
assert isinstance(qkv_proj.scheme, CompressedTensorsW4A16Sparse24)
128
assert qkv_proj.weight_packed.dtype is torch.int32
129
123
- sampling_params = SamplingParams()
124
- output = llm.generate("Hello world!", sampling_params=sampling_params)
130
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
131
assert output
132
133
@@ -142,6 +148,5 @@ def test_compressed_tensors_fp8(vllm_runner):
148
assert len(qkv_proj.input_scale.shape) == 0
149
assert len(qkv_proj.weight_scale.shape) == 0
150
145
- sampling_params = SamplingParams()
146
- output = llm.generate("Hello world!", sampling_params=sampling_params)
151
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
152
assert output
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+69
-82
@@ -6,9 +6,10 @@ from torch.nn import Parameter
6
from vllm import _custom_ops as ops
7
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
CompressedTensorsScheme)
9
-from vllm.model_executor.layers.quantization.gptq_marlin import (
10
- GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, GPTQMarlinState,
11
- marlin_permute_scales)
9
+from vllm.model_executor.layers.quantization.utils.marlin_utils import (
10
+ apply_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
11
+ marlin_permute_scales, replace_tensor, verify_marlin_supported,
12
+ verify_marlin_supports_shape)
13
from vllm.model_executor.utils import set_weight_attrs
14
15
__all__ = ["CompressedTensorsWNA16"]
@@ -22,29 +23,40 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
23
num_bits: int,
24
group_size: Optional[int] = None):
25
self.num_bits = num_bits
26
+ self.pack_factor = 32 // self.num_bits
27
self.strategy = strategy
26
- self.group_size = group_size
28
28
- if self.strategy == "group" and self.group_size is None:
29
- raise ValueError(
30
- "group_size must be given when using strategy group")
29
+ self.group_size: int
30
+ if group_size is None:
31
+ if self.strategy != "channel":
32
+ raise ValueError(
33
+ "Marlin kernels require group quantization or "
34
+ "channelwise quantization, but found no group "
35
+ "size and strategy is not channelwise.")
36
+ self.group_size = -1
37
+ else:
38
+ self.group_size = group_size
39
32
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
33
- pass
40
+ # Verify supported on platform.
41
+ verify_marlin_supported(num_bits=self.num_bits,
42
+ group_size=self.group_size,
43
+ is_sym=True)
44
45
def create_weights(self, layer: torch.nn.Module, input_size: int,
46
output_partition_sizes: List[int],
47
input_size_per_partition: int,
48
params_dtype: torch.dtype, weight_loader: Callable,
49
**kwargs):
40
-
41
- pack_factor = 32 // self.num_bits
50
output_size_per_partition = sum(output_partition_sizes)
51
44
- if self.group_size is not None:
45
- group_size = self.group_size
46
- else:
47
- group_size = input_size
52
+ # If group_size is -1, we are in channelwise case.
53
+ group_size = input_size if self.group_size == -1 else self.group_size
54
+
55
+ verify_marlin_supports_shape(
56
+ output_size_per_partition=output_size_per_partition,
57
+ input_size_per_partition=input_size_per_partition,
58
+ input_size=input_size,
59
+ group_size=group_size)
60
61
weight_scale_dim = None
62
scales_and_zp_size = input_size // group_size
@@ -57,7 +69,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
69
weight = Parameter(
70
torch.empty(
71
output_size_per_partition,
60
- input_size_per_partition // pack_factor,
72
+ input_size_per_partition // self.pack_factor,
73
dtype=torch.int32,
74
),
75
requires_grad=False,
@@ -68,7 +80,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
80
"input_dim": 1,
81
"output_dim": 0,
82
"packed_dim": 1,
71
- "pack_factor": pack_factor,
83
+ "pack_factor": self.pack_factor,
84
"weight_loader": weight_loader
85
})
86
layer.register_parameter("weight_packed", weight)
@@ -103,73 +115,48 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
115
116
layer.input_size_per_partition = input_size_per_partition
117
layer.output_size_per_partition = output_size_per_partition
106
-
118
layer.input_size = input_size
108
- layer.marlin_state = GPTQMarlinState.REPACK
109
- layer.is_k_full = True
119
layer.group_size = group_size
120
112
- max_workspace_size = (
113
- output_size_per_partition //
114
- GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL
115
-
116
- workspace = torch.zeros(max_workspace_size,
117
- dtype=torch.int,
118
- requires_grad=False)
119
- layer.workspace = workspace
121
+ # Checkpoints are serialized in compressed-tensors format, which is
122
+ # different from marlin format. Handle repacking here.
123
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
124
+ device = layer.weight_packed.device
125
+
126
+ # Allocate marlin workspace.
127
+ layer.workspace = marlin_make_workspace(
128
+ layer.output_size_per_partition, device)
129
+
130
+ # Act-order not supported in compressed-tensors yet, so set to empty.
131
+ layer.g_idx = marlin_make_empty_g_idx(device)
132
+ layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
133
+
134
+ # Repack weights from compressed-tensors format to marlin format.
135
+ marlin_qweight = ops.gptq_marlin_repack(
136
+ layer.weight_packed.t().contiguous(),
137
+ perm=layer.g_idx_sort_indices,
138
+ size_k=layer.input_size_per_partition,
139
+ size_n=layer.output_size_per_partition,
140
+ num_bits=self.num_bits)
141
+ replace_tensor(layer, "weight_packed", marlin_qweight)
142
+
143
+ # Permute scales from compressed-tensors format to marlin format.
144
+ marlin_scales = marlin_permute_scales(
145
+ layer.weight_scale.squeeze().t().contiguous(),
146
+ size_k=layer.input_size_per_partition,
147
+ size_n=layer.output_size_per_partition,
148
+ group_size=layer.group_size)
149
+ replace_tensor(layer, "weight_scale", marlin_scales)
150
151
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
122
- reshaped_x = x.reshape(-1, x.shape[-1])
123
-
124
- size_m = reshaped_x.shape[0]
125
- part_size_n = layer.output_size_per_partition
126
- part_size_k = layer.input_size_per_partition
127
-
128
- out_shape = x.shape[:-1] + (part_size_n, )
129
-
130
- if layer.marlin_state == GPTQMarlinState.REPACK:
131
- layer.marlin_state = GPTQMarlinState.READY
132
-
133
- # Newly generated tensors need to replace existing tensors that are
134
- # already registered as parameters by vLLM (and won't be freed)
135
- def replace_tensor(name, new_t):
136
- # It is important to use resize_() here since it ensures
137
- # the same buffer is reused
138
- getattr(layer, name).resize_(new_t.shape)
139
- getattr(layer, name).copy_(new_t)
140
- del new_t
141
-
142
- cur_device = layer.weight_packed.device
143
-
144
- # Reset g_idx related tensors
145
- layer.g_idx = Parameter(torch.empty(0,
146
- dtype=torch.int,
147
- device=cur_device),
148
- requires_grad=False)
149
- layer.g_idx_sort_indices = Parameter(torch.empty(
150
- 0, dtype=torch.int, device=cur_device),
151
- requires_grad=False)
152
-
153
- # Repack weights
154
- marlin_qweight = ops.gptq_marlin_repack(
155
- layer.weight_packed.t().contiguous(), layer.g_idx_sort_indices,
156
- part_size_k, part_size_n, self.num_bits)
157
-
158
- replace_tensor("weight_packed", marlin_qweight)
159
-
160
- # Permute scales
161
- scales_size_k = part_size_k
162
- scales_size_n = part_size_n
163
-
164
- marlin_scales = marlin_permute_scales(
165
- layer.weight_scale.squeeze().t().contiguous(), scales_size_k,
166
- scales_size_n, layer.group_size, self.num_bits)
167
- replace_tensor("weight_scale", marlin_scales)
168
-
169
- output = ops.gptq_marlin_gemm(reshaped_x, layer.weight_packed,
170
- layer.weight_scale, layer.g_idx,
171
- layer.g_idx_sort_indices,
172
- layer.workspace, self.num_bits, size_m,
173
- part_size_n, part_size_k,
174
- layer.is_k_full)
175
- return output.reshape(out_shape)
152
+ return apply_marlin_linear(
153
+ input=x,
154
+ weight=layer.weight_packed,
155
+ weight_scale=layer.weight_scale,
156
+ g_idx=layer.g_idx,
157
+ g_idx_sort_indices=layer.g_idx_sort_indices,
158
+ workspace=layer.workspace,
159
+ num_bits=self.num_bits,
160
+ output_size_per_partition=layer.output_size_per_partition,
161
+ input_size_per_partition=layer.input_size_per_partition,
162
+ is_k_full=True)
vllm/model_executor/layers/quantization/fp8.py
+1
-1
@@ -11,7 +11,7 @@ from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
11
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig, QuantizeMethodBase)
14
-from vllm.model_executor.layers.quantization.utils.marlin_utils import (
14
+from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
15
apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
16
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
17
all_close_1d, apply_fp8_linear, create_per_tensor_scale_param,
vllm/model_executor/layers/quantization/gptq_marlin.py
+67
-196
@@ -1,5 +1,3 @@
1
-import enum
2
-from enum import Enum
1
from typing import Any, Dict, List, Optional
2
3
import torch
@@ -12,46 +10,14 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10
from vllm.model_executor.layers.quantization.base_config import (
11
QuantizationConfig)
12
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15
- GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_K,
16
- GPTQ_MARLIN_MIN_THREAD_N, GPTQ_MARLIN_SUPPORTED_GROUP_SIZES,
17
- GPTQ_MARLIN_SUPPORTED_NUM_BITS, GPTQ_MARLIN_SUPPORTED_SYM,
18
- GPTQ_MARLIN_TILE)
13
+ check_marlin_supported, marlin_make_empty_g_idx, marlin_make_workspace,
14
+ marlin_permute_scales, marlin_sort_g_idx, replace_tensor,
15
+ verify_marlin_supported, verify_marlin_supports_shape)
16
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
20
-from vllm.platforms import current_platform
17
18
logger = init_logger(__name__)
19
20
25
-# Permutations for Marlin scale shuffling
26
-def get_scale_perms(num_bits: int):
27
- scale_perm: List[int] = []
28
- for i in range(8):
29
- scale_perm.extend([i + 8 * j for j in range(8)])
30
- scale_perm_single: List[int] = []
31
- for i in range(4):
32
- scale_perm_single.extend(
33
- [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
34
- return scale_perm, scale_perm_single
35
-
36
-
37
-def get_pack_factor(num_bits: int):
38
- assert (num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS
39
- ), f"Unsupported num_bits = {num_bits}"
40
- return 32 // num_bits
41
-
42
-
43
-def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
44
- group_size: int, num_bits: int):
45
- scale_perm, scale_perm_single = get_scale_perms(num_bits)
46
- if group_size < size_k and group_size != -1:
47
- s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
48
- else:
49
- s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single]
50
- s = s.reshape((-1, size_n)).contiguous()
51
-
52
- return s
53
-
54
-
21
class GPTQMarlinConfig(QuantizationConfig):
22
"""Config class for GPTQ Marlin"""
23
@@ -63,33 +29,16 @@ class GPTQMarlinConfig(QuantizationConfig):
29
desc_act = False
30
31
self.weight_bits = weight_bits
32
+ self.pack_factor = 32 // self.weight_bits # packed into int32
33
self.group_size = group_size
34
self.desc_act = desc_act
35
self.is_sym = is_sym
36
self.lm_head_quantized = lm_head_quantized
37
71
- # Verify
72
- if self.weight_bits not in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
73
- raise ValueError(
74
- f"Marlin does not support weight_bits = {self.weight_bits}. "
75
- f"Only weight_bits = {GPTQ_MARLIN_SUPPORTED_NUM_BITS} "
76
- "are supported.")
77
- if self.group_size not in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES:
78
- raise ValueError(
79
- f"Marlin does not support group_size = {self.group_size}. "
80
- f"Only group_sizes = {GPTQ_MARLIN_SUPPORTED_GROUP_SIZES} "
81
- "are supported.")
82
- if self.is_sym not in GPTQ_MARLIN_SUPPORTED_SYM:
83
- raise ValueError(
84
- f"Marlin does not support is_sym = {self.is_sym}. "
85
- f"Only sym = {GPTQ_MARLIN_SUPPORTED_SYM} are supported.")
86
-
87
- # Init
88
- self.pack_factor = get_pack_factor(weight_bits)
89
- self.tile_size = GPTQ_MARLIN_TILE
90
- self.min_thread_n = GPTQ_MARLIN_MIN_THREAD_N
91
- self.min_thread_k = GPTQ_MARLIN_MIN_THREAD_K
92
- self.max_parallel = GPTQ_MARLIN_MAX_PARALLEL
38
+ # Verify supported on platform.
39
+ verify_marlin_supported(num_bits=self.weight_bits,
40
+ group_size=self.group_size,
41
+ is_sym=self.is_sym)
42
43
def __repr__(self) -> str:
44
return (f"GPTQMarlinConfig(weight_bits={self.weight_bits}, "
@@ -168,21 +117,10 @@ class GPTQMarlinConfig(QuantizationConfig):
117
or desc_act is None):
118
return False
119
171
- # If the capability of the device is too low, cannot convert.
172
- major, minor = current_platform.get_device_capability()
173
- device_capability = major * 10 + minor
174
- if device_capability < cls.get_min_capability():
175
- return False
176
-
177
- # Otherwise, can convert if model satisfies marlin constraints.
178
- return (num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS
179
- and group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES
180
- and sym in GPTQ_MARLIN_SUPPORTED_SYM)
181
-
182
-
183
-class GPTQMarlinState(Enum):
184
- REPACK = enum.auto()
185
- READY = enum.auto()
120
+ return check_marlin_supported(num_bits=num_bits,
121
+ group_size=group_size,
122
+ is_sym=sym,
123
+ min_capability=cls.get_min_capability())
124
125
126
class GPTQMarlinLinearMethod(LinearMethodBase):
@@ -206,6 +144,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
144
**extra_weight_attrs,
145
) -> None:
146
del output_size
147
+ output_size_per_partition = sum(output_partition_sizes)
148
149
# Normalize group_size
150
if self.quant_config.group_size != -1:
@@ -213,31 +152,11 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
152
else:
153
group_size = input_size
154
216
- # Validate dtype
217
- if params_dtype not in [torch.float16, torch.bfloat16]:
218
- raise ValueError(f"The params dtype must be float16 "
219
- f"or bfloat16, but got {params_dtype}")
220
-
221
- # Validate output_size_per_partition
222
- output_size_per_partition = sum(output_partition_sizes)
223
- if output_size_per_partition % self.quant_config.min_thread_n != 0:
224
- raise ValueError(
225
- f"Weight output_size_per_partition = "
226
- f"{output_size_per_partition} is not divisible by "
227
- f" min_thread_n = {self.quant_config.min_thread_n}.")
228
-
229
- # Validate input_size_per_partition
230
- if input_size_per_partition % self.quant_config.min_thread_k != 0:
231
- raise ValueError(
232
- f"Weight input_size_per_partition = "
233
- f"{input_size_per_partition} is not divisible "
234
- f"by min_thread_k = {self.quant_config.min_thread_k}.")
235
-
236
- if (group_size < input_size
237
- and input_size_per_partition % group_size != 0):
238
- raise ValueError(
239
- f"Weight input_size_per_partition = {input_size_per_partition}"
240
- f" is not divisible by group_size = {group_size}.")
155
+ verify_marlin_supports_shape(
156
+ output_size_per_partition=output_size_per_partition,
157
+ input_size_per_partition=input_size_per_partition,
158
+ input_size=input_size,
159
+ group_size=group_size)
160
161
# Detect sharding of scales/zp
162
@@ -303,11 +222,6 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
222
},
223
)
224
306
- g_idx_sort_indices = torch.empty(
307
- g_idx.shape,
308
- dtype=torch.int32,
309
- )
310
-
225
# Scales
226
scales = Parameter(
227
torch.empty(
@@ -347,25 +261,50 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
261
},
262
)
263
350
- # Allocate marlin workspace
351
- max_workspace_size = (
352
- output_size_per_partition //
353
- self.quant_config.min_thread_n) * self.quant_config.max_parallel
354
- workspace = torch.zeros(max_workspace_size,
355
- dtype=torch.int,
356
- requires_grad=False)
357
-
264
layer.register_parameter("qweight", qweight)
265
layer.register_parameter("g_idx", g_idx)
266
layer.register_parameter("scales", scales)
267
layer.register_parameter("qzeros", qzeros)
362
- layer.g_idx_sort_indices = g_idx_sort_indices
363
- layer.workspace = workspace
268
layer.input_size_per_partition = input_size_per_partition
269
layer.output_size_per_partition = output_size_per_partition
270
layer.input_size = input_size
271
layer.is_k_full = is_k_full
368
- layer.marlin_state = GPTQMarlinState.REPACK
272
+
273
+ # Checkpoints are serialized in AutoGPTQ format, which is different from the
274
+ # marlin format. This function is called after the weights are loaded.
275
+ # Here, we handle the repacking, including the activation reordering case.
276
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
277
+ device = layer.qweight.device
278
+ # Allocate marlin workspace
279
+ layer.workspace = marlin_make_workspace(
280
+ layer.output_size_per_partition, device)
281
+
282
+ # Handle sorting for activation reordering if needed.
283
+ if self.quant_config.desc_act:
284
+ g_idx, g_idx_sort_indices = marlin_sort_g_idx(layer.g_idx)
285
+ layer.g_idx_sort_indices = g_idx_sort_indices
286
+ replace_tensor(layer, "g_idx", g_idx)
287
+ else:
288
+ layer.g_idx = marlin_make_empty_g_idx(device)
289
+ layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
290
+
291
+ # Repack weights from autogptq format to marlin format.
292
+ marlin_qweight = ops.gptq_marlin_repack(
293
+ layer.qweight,
294
+ perm=layer.g_idx_sort_indices,
295
+ size_k=layer.input_size_per_partition,
296
+ size_n=layer.output_size_per_partition,
297
+ num_bits=self.quant_config.weight_bits)
298
+ replace_tensor(layer, "qweight", marlin_qweight)
299
+
300
+ # Permute scales from autogptq format to marlin format.
301
+ marlin_scales = marlin_permute_scales(
302
+ layer.scales,
303
+ size_k=(layer.input_size if self.quant_config.desc_act else
304
+ layer.input_size_per_partition),
305
+ size_n=layer.output_size_per_partition,
306
+ group_size=self.quant_config.group_size)
307
+ replace_tensor(layer, "scales", marlin_scales)
308
309
def apply(
310
self,
@@ -374,87 +313,19 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
313
bias: Optional[torch.Tensor] = None,
314
) -> torch.Tensor:
315
reshaped_x = x.reshape(-1, x.shape[-1])
377
-
378
- size_m = reshaped_x.shape[0]
379
- part_size_n = layer.output_size_per_partition
380
- part_size_k = layer.input_size_per_partition
381
- full_size_k = layer.input_size
382
-
383
- out_shape = x.shape[:-1] + (part_size_n, )
384
-
385
- if layer.marlin_state == GPTQMarlinState.REPACK:
386
- layer.marlin_state = GPTQMarlinState.READY
387
-
388
- # Newly generated tensors need to replace existing tensors that are
389
- # already registered as parameters by vLLM (and won't be freed)
390
- def replace_tensor(name, new_t):
391
- # It is important to use resize_() here since it ensures
392
- # the same buffer is reused
393
- getattr(layer, name).resize_(new_t.shape)
394
- getattr(layer, name).copy_(new_t)
395
- del new_t
396
-
397
- cur_device = layer.qweight.device
398
-
399
- # Process act_order
400
- if self.quant_config.desc_act:
401
- # Get sorting based on g_idx
402
- g_idx_sort_indices = torch.argsort(layer.g_idx).to(torch.int)
403
-
404
- sorted_g_idx = layer.g_idx[g_idx_sort_indices]
405
-
406
- replace_tensor("g_idx", sorted_g_idx)
407
- replace_tensor("g_idx_sort_indices", g_idx_sort_indices)
408
-
409
- else:
410
- # Reset g_idx related tensors
411
- layer.g_idx = Parameter(
412
- torch.empty(0, dtype=torch.int, device=cur_device),
413
- requires_grad=False,
414
- )
415
- layer.g_idx_sort_indices = Parameter(
416
- torch.empty(0, dtype=torch.int, device=cur_device),
417
- requires_grad=False,
418
- )
419
-
420
- # Repack weights
421
- marlin_qweight = ops.gptq_marlin_repack(
422
- layer.qweight,
423
- layer.g_idx_sort_indices,
424
- part_size_k,
425
- part_size_n,
426
- self.quant_config.weight_bits,
427
- )
428
- replace_tensor("qweight", marlin_qweight)
429
-
430
- # Permute scales
431
- scales_size_k = part_size_k
432
- scales_size_n = part_size_n
433
- if self.quant_config.desc_act:
434
- scales_size_k = full_size_k
435
-
436
- marlin_scales = marlin_permute_scales(
437
- layer.scales,
438
- scales_size_k,
439
- scales_size_n,
440
- self.quant_config.group_size,
441
- self.quant_config.weight_bits,
442
- )
443
- replace_tensor("scales", marlin_scales)
444
-
445
- output = ops.gptq_marlin_gemm(
446
- reshaped_x,
447
- layer.qweight,
448
- layer.scales,
449
- layer.g_idx,
450
- layer.g_idx_sort_indices,
451
- layer.workspace,
452
- self.quant_config.weight_bits,
453
- size_m,
454
- part_size_n,
455
- part_size_k,
456
- layer.is_k_full,
457
- )
316
+ out_shape = x.shape[:-1] + (layer.output_size_per_partition, )
317
+
318
+ output = ops.gptq_marlin_gemm(reshaped_x,
319
+ layer.qweight,
320
+ layer.scales,
321
+ g_idx=layer.g_idx,
322
+ perm=layer.g_idx_sort_indices,
323
+ workspace=layer.workspace,
324
+ num_bits=self.quant_config.weight_bits,
325
+ size_m=reshaped_x.shape[0],
326
+ size_n=layer.output_size_per_partition,
327
+ size_k=layer.input_size_per_partition,
328
+ is_k_full=layer.is_k_full)
329
330
if bias is not None:
331
output.add_(bias) # In-place add
vllm/model_executor/layers/quantization/utils/marlin_24_perms.py
deleted
-60
@@ -1,60 +0,0 @@
1
-"""This file is used for /tests and /benchmarks"""
2
-from typing import Dict, List
3
-
4
-import numpy
5
-import torch
6
-
7
-
8
-# Precompute permutations for Marlin24 weight and scale shuffling # noqa: E501
9
-#
10
-# Marlin works on [16*2,64] tiles. The goal of the permutations is to reorder the weight data so that it is compatible noqa: # noqa: E501
11
-# with the tensor-core format that is described here:
12
-# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
13
-#
14
-# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
15
-# (without the need to use ldmatrix instructions) # noqa: E501
16
-def get_perms_24(num_bits: int):
17
- perm_list: List[int] = []
18
- for i in range(32):
19
- perm1: List[int] = []
20
- col = i // 4
21
- col_o = col // 2
22
- for block in [0, 1]:
23
- for row in [
24
- 2 * (i % 4),
25
- 2 * (i % 4) + 1,
26
- 2 * (i % 4 + 4),
27
- 2 * (i % 4 + 4) + 1,
28
- ]:
29
- perm1.append(16 * row + col_o * 256 + 8 * (col % 2) +
30
- 4 * block)
31
- for j in range(4):
32
- perm_list.extend([p + 1 * j for p in perm1])
33
- perm = numpy.array(perm_list)
34
-
35
- if num_bits == 4:
36
- interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
37
- elif num_bits == 8:
38
- interleave = numpy.array([0, 2, 1, 3])
39
- else:
40
- raise ValueError("num_bits must be 4 or 8, got {}".format(num_bits))
41
-
42
- perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
43
- perm = torch.from_numpy(perm)
44
- scale_perm: List[int] = []
45
- for i in range(8):
46
- scale_perm.extend([i * 8 + j for j in [0, 4, 1, 5, 2, 6, 3, 7]])
47
- scale_perm_single: List[int] = []
48
- for i in range(8):
49
- scale_perm_single.extend([8 * i + j for j in [0, 1, 2, 3, 4, 5, 6, 7]])
50
- return perm, scale_perm, scale_perm_single
51
-
52
-
53
-marlin_24_perm: Dict[int, torch.Tensor] = {}
54
-marlin_24_scale_perm: Dict[int, List[int]] = {}
55
-marlin_24_scale_perm_single: Dict[int, List[int]] = {}
56
-for num_bits in [4, 8]:
57
- perm_24, scale_perm_24, scale_perm_single_24 = get_perms_24(num_bits)
58
- marlin_24_perm[num_bits] = perm_24
59
- marlin_24_scale_perm[num_bits] = scale_perm_24
60
- marlin_24_scale_perm_single[num_bits] = scale_perm_single_24
vllm/model_executor/layers/quantization/utils/marlin_perms.py
deleted
-60
@@ -1,60 +0,0 @@
1
-"""This file is used for /tests and /benchmarks"""
2
-from typing import Dict, List
3
-
4
-import numpy
5
-import torch
6
-
7
-
8
-# Precompute permutations for Marlin weight and scale shuffling # noqa: E501
9
-#
10
-# Marlin works on [16,64] tiles. The goal of the permutations is to reorder the weight data so that it is compatible noqa: # noqa: E501
11
-# with the tensor-core format that is described here:
12
-# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
13
-#
14
-# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
15
-# (without the need to use ldmatrix instructions) # noqa: E501
16
-def get_perms(num_bits: int):
17
- perm_list: List[int] = []
18
- for i in range(32):
19
- perm1: List[int] = []
20
- col = i // 4
21
- for block in [0, 1]:
22
- for row in [
23
- 2 * (i % 4),
24
- 2 * (i % 4) + 1,
25
- 2 * (i % 4 + 4),
26
- 2 * (i % 4 + 4) + 1,
27
- ]:
28
- perm1.append(16 * row + col + 8 * block)
29
- for j in range(4):
30
- perm_list.extend([p + 256 * j for p in perm1])
31
-
32
- perm = numpy.array(perm_list)
33
-
34
- if num_bits == 4:
35
- interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
36
- elif num_bits == 8:
37
- interleave = numpy.array([0, 2, 1, 3])
38
- else:
39
- raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
40
-
41
- perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
42
- perm = torch.from_numpy(perm)
43
- scale_perm: List[int] = []
44
- for i in range(8):
45
- scale_perm.extend([i + 8 * j for j in range(8)])
46
- scale_perm_single: List[int] = []
47
- for i in range(4):
48
- scale_perm_single.extend(
49
- [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
50
- return perm, scale_perm, scale_perm_single
51
-
52
-
53
-marlin_perm: Dict[int, torch.Tensor] = {}
54
-marlin_scale_perm: Dict[int, List[int]] = {}
55
-marlin_scale_perm_single: Dict[int, List[int]] = {}
56
-for num_bits in [4, 8]:
57
- perm, scale_perm, scale_perm_single = get_perms(num_bits)
58
- marlin_perm[num_bits] = perm
59
- marlin_scale_perm[num_bits] = scale_perm
60
- marlin_scale_perm_single[num_bits] = scale_perm_single
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+133
-306
@@ -1,21 +1,9 @@
1
-"""This file is used for /tests and /benchmarks"""
2
-import random
3
-from typing import Optional
1
+from typing import List, Optional, Tuple
2
5
-import numpy
3
import torch
4
5
from vllm import _custom_ops as ops
9
-from vllm.model_executor.layers.quantization.utils.format_24 import (
10
- mask_creator, sparse_semi_structured_from_dense_cutlass)
11
-from vllm.model_executor.layers.quantization.utils.marlin_24_perms import (
12
- marlin_24_perm, marlin_24_scale_perm, marlin_24_scale_perm_single)
13
-from vllm.model_executor.layers.quantization.utils.marlin_perms import (
14
- marlin_perm, marlin_scale_perm, marlin_scale_perm_single)
15
-from vllm.model_executor.layers.quantization.utils.quant_utils import (
16
- get_pack_factor, quantize_weights, sort_weights)
6
from vllm.platforms import current_platform
18
-from vllm.utils import print_warning_once
7
8
GPTQ_MARLIN_TILE = 16
9
GPTQ_MARLIN_MIN_THREAD_N = 64
@@ -25,135 +13,110 @@ GPTQ_MARLIN_MAX_PARALLEL = 16
13
GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4, 8]
14
GPTQ_MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
15
GPTQ_MARLIN_SUPPORTED_SYM = [True]
28
-
29
-
30
-def is_marlin_supported():
31
- capability = current_platform.get_device_capability()
32
- return capability[0] >= 8
33
-
34
-
35
-def apply_fp8_marlin_linear(
36
- input: torch.Tensor,
37
- weight: torch.Tensor,
38
- weight_scale: torch.Tensor,
39
- workspace: torch.Tensor,
40
- size_n: int,
41
- size_k: int,
42
- bias: Optional[torch.Tensor],
43
-) -> torch.Tensor:
44
- # For GPUs that lack FP8 hardware support, we can leverage the
45
- # Marlin kernel for fast weight-only FP8 quantization
46
-
47
- reshaped_x = input.reshape(-1, input.shape[-1])
48
- out_shape = input.shape[:-1] + (size_n, )
49
-
50
- output = ops.fp8_marlin_gemm(
51
- a=reshaped_x,
52
- b_q_weight=weight,
53
- b_scales=weight_scale,
54
- workspace=workspace,
55
- num_bits=8,
56
- size_m=reshaped_x.shape[0],
57
- size_n=size_n,
58
- size_k=size_k,
59
- )
60
-
61
- if bias is not None:
62
- output.add_(bias) # In-place add
63
-
64
- return output.reshape(out_shape)
65
-
66
-
67
-def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
68
- print_warning_once(
69
- "Your GPU does not have native support for FP8 computation but "
70
- "FP8 quantization is being used. Weight-only FP8 compression will "
71
- "be used leveraging the Marlin kernel. This may degrade "
72
- "performance for compute-heavy workloads.")
73
-
74
- part_size_n = layer.output_size_per_partition
75
- part_size_k = layer.input_size_per_partition
76
-
77
- device = layer.weight.device
78
-
79
- # WEIGHTS
80
- # Repack weights to gptq format (packed int32 elements)
81
- packed_gptq_qweight = pack_fp8_to_int32(layer.weight)
82
-
83
- # Repack weights to marlin format
84
- marlin_qweight = ops.gptq_marlin_repack(
85
- b_q_weight=packed_gptq_qweight,
86
- perm=torch.empty(0, dtype=torch.int, device=device),
87
- size_k=part_size_k,
88
- size_n=part_size_n,
89
- num_bits=8,
90
- )
91
- layer.weight = torch.nn.Parameter(marlin_qweight, requires_grad=False)
92
-
93
- # WEIGHT SCALES
94
- # Currently Marlin doesn't support per-tensor scales, so we
95
- # expand it to channelwise
96
- scales = layer.weight_scale.repeat(1, part_size_n).to(
97
- layer.orig_dtype).to(device)
98
- # Permute scales
99
- num_bits = 8
100
- marlin_scales = marlin_permute_scales(
101
- s=scales,
102
- size_k=part_size_k,
103
- size_n=part_size_n,
104
- group_size=-1,
105
- scale_perm=marlin_scale_perm[num_bits],
106
- scale_perm_single=marlin_scale_perm_single[num_bits])
107
- layer.weight_scale = torch.nn.Parameter(marlin_scales, requires_grad=False)
108
-
109
- # Allocate marlin workspace
110
- max_workspace_size = (part_size_n //
16
+GTPQ_MARLIN_UNSUPPORTED_GROUP_SIZE_ACT_ORDER = [-1]
17
+
18
+
19
+def check_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
20
+ min_capability: int) -> bool:
21
+
22
+ # If the capability of the device is too low, cannot convert.
23
+ major, minor = current_platform.get_device_capability()
24
+ device_capability = major * 10 + minor
25
+ if device_capability < min_capability:
26
+ return False
27
+
28
+ return (device_capability >= min_capability
29
+ and num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS
30
+ and group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES
31
+ and is_sym in GPTQ_MARLIN_SUPPORTED_SYM)
32
+
33
+
34
+def verify_marlin_supported(num_bits: int, group_size: Optional[int],
35
+ is_sym: bool) -> None:
36
+
37
+ if num_bits not in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
38
+ raise ValueError(
39
+ f"Marlin does not support weight_bits = {num_bits}. "
40
+ f"Only weight_bits = {GPTQ_MARLIN_SUPPORTED_NUM_BITS} "
41
+ "are supported.")
42
+ if (group_size is None
43
+ or group_size not in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES):
44
+ raise ValueError(
45
+ f"Marlin does not support group_size = {group_size}. "
46
+ f"Only group_sizes = {GPTQ_MARLIN_SUPPORTED_GROUP_SIZES} "
47
+ "are supported.")
48
+ if is_sym not in GPTQ_MARLIN_SUPPORTED_SYM:
49
+ raise ValueError(
50
+ f"Marlin does not support is_sym = is_sym. "
51
+ f"Only sym = {GPTQ_MARLIN_SUPPORTED_SYM} are supported.")
52
+
53
+
54
+def verify_marlin_supports_shape(output_size_per_partition: int,
55
+ input_size_per_partition: int,
56
+ input_size: int, group_size: int) -> None:
57
+
58
+ # Validate output_size_per_partition
59
+ if output_size_per_partition % GPTQ_MARLIN_MIN_THREAD_N != 0:
60
+ raise ValueError(f"Weight output_size_per_partition = "
61
+ f"{output_size_per_partition} is not divisible by "
62
+ f" min_thread_n = {GPTQ_MARLIN_MIN_THREAD_N}. "
63
+ "Consider reducing tensor_parallel_size or running "
64
+ "with --quantization gptq.")
65
+
66
+ # Validate input_size_per_partition
67
+ if input_size_per_partition % GPTQ_MARLIN_MIN_THREAD_K != 0:
68
+ raise ValueError(f"Weight input_size_per_partition = "
69
+ f"{input_size_per_partition} is not divisible "
70
+ f"by min_thread_k = {GPTQ_MARLIN_MIN_THREAD_K}. "
71
+ "Consider reducing tensor_parallel_size or running "
72
+ "with --quantization gptq.")
73
+
74
+ if (group_size < input_size
75
+ and input_size_per_partition % group_size != 0):
76
+ raise ValueError(
77
+ f"Weight input_size_per_partition = {input_size_per_partition}"
78
+ f" is not divisible by group_size = {group_size}."
79
+ "Consider reducing tensor_parallel_size or running "
80
+ "with --quantization gptq.")
81
+
82
+
83
+def marlin_make_workspace(output_size_per_partition: int,
84
+ device: torch.device) -> torch.Tensor:
85
+ max_workspace_size = (output_size_per_partition //
86
GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL
112
- workspace = torch.zeros(max_workspace_size,
113
- dtype=torch.int,
114
- device=device,
115
- requires_grad=False)
116
-
117
- layer.workspace = workspace
118
-
119
-
120
-def marlin_permute_weights(q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE):
121
- assert q_w.shape == (size_k, size_n)
122
- assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
123
- assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
124
-
125
- # Permute weights to 16x64 marlin tiles
126
- q_w = q_w.reshape((size_k // tile, tile, size_n // tile, tile))
127
- q_w = q_w.permute((0, 2, 1, 3))
128
- q_w = q_w.reshape((size_k // tile, size_n * tile))
87
130
- q_w = q_w.reshape((-1, perm.numel()))[:, perm].reshape(q_w.shape)
88
+ return torch.zeros(max_workspace_size,
89
+ dtype=torch.int,
90
+ device=device,
91
+ requires_grad=False)
92
132
- return q_w
93
94
+def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:
95
+ return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
96
+ requires_grad=False)
97
135
-def marlin_weights(q_w, size_k, size_n, num_bits, perm):
136
- # Permute
137
- q_w = marlin_permute_weights(q_w, size_k, size_n, perm)
98
139
- # Pack
140
- pack_factor = get_pack_factor(num_bits)
141
- orig_device = q_w.device
99
+def marlin_sort_g_idx(
100
+ g_idx: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
101
+ g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)
102
+ return g_idx[g_idx_sort_indices], g_idx_sort_indices
103
143
- q_w = q_w.cpu().numpy().astype(numpy.uint32)
104
145
- q_packed = numpy.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
146
- dtype=numpy.uint32)
147
- for i in range(pack_factor):
148
- q_packed |= q_w[:, i::pack_factor] << num_bits * i
105
+def get_scale_perms():
106
+ scale_perm: List[int] = []
107
+ for i in range(8):
108
+ scale_perm.extend([i + 8 * j for j in range(8)])
109
+ scale_perm_single: List[int] = []
110
+ for i in range(4):
111
+ scale_perm_single.extend(
112
+ [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
113
+ return scale_perm, scale_perm_single
114
150
- q_packed = torch.from_numpy(q_packed.astype(numpy.int32)).to(orig_device)
115
152
- return q_packed
116
+def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
117
+ group_size: int) -> torch.Tensor:
118
154
-
155
-def marlin_permute_scales(s, size_k, size_n, group_size, scale_perm,
156
- scale_perm_single):
119
+ scale_perm, scale_perm_single = get_scale_perms()
120
if group_size < size_k and group_size != -1:
121
s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
122
else:
@@ -163,180 +126,44 @@ def marlin_permute_scales(s, size_k, size_n, group_size, scale_perm,
126
return s
127
128
166
-def marlin_quantize(
167
- w: torch.Tensor,
168
- num_bits: int,
169
- group_size: int,
170
- act_order: bool,
171
-):
172
- size_k, size_n = w.shape
173
-
174
- # Normalize group_size
175
- if group_size == -1:
176
- group_size = size_k
177
- assert group_size <= size_k
178
-
179
- # Quantize (and apply act_order if provided)
180
- w_ref, q_w, s, g_idx, rand_perm = quantize_weights(w, num_bits, group_size,
181
- act_order)
182
-
183
- # For act_order, sort the "weights" and "g_idx" so that group ids are
184
- # increasing
185
- sort_indices = torch.empty(0, dtype=torch.int, device=w.device)
186
- if act_order:
187
- q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
188
-
189
- # Reformat to marlin
190
- marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits,
191
- marlin_perm[num_bits])
192
- marlin_s = marlin_permute_scales(s, size_k, size_n, group_size,
193
- marlin_scale_perm[num_bits],
194
- marlin_scale_perm_single[num_bits])
195
-
196
- # Create result
197
- res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm]
198
- for i in range(len(res_list)):
199
- res_list[i] = res_list[i].to(w.device)
200
-
201
- return res_list
202
-
203
-
204
-def inject_24(w, size_k, size_n):
205
- assert w.shape == (size_k, size_n)
206
-
207
- mask = mask_creator(w.t()).t().cuda().bool()
208
-
209
- return (mask * w).contiguous(), mask.contiguous()
210
-
211
-
212
-def check_24(w, num_rows_to_sample=50, _verbose=False):
213
- BLOCK_SIZE = 4
214
- MAX_NON_ZEROS = 2
215
-
216
- w = w.t().contiguous()
217
-
218
- print("check_24: w.shape = {}".format(w.shape))
219
-
220
- num_rows, num_cols = w.shape
221
- sampled_row_idxs = random.choices(range(num_rows), k=num_rows_to_sample)
222
- if _verbose:
223
- print(f"Sampled row idxs = {sampled_row_idxs}")
224
-
225
- total_segments = 0
226
- non_24_segments = 0
227
- for i in sampled_row_idxs:
228
- for j in range(0, num_cols - BLOCK_SIZE, BLOCK_SIZE):
229
- total_segments += 1
230
- block = w[i, j:j + BLOCK_SIZE]
231
- num_nonzero = torch.count_nonzero(block)
232
- if num_nonzero > MAX_NON_ZEROS:
233
- print("i = {} j = {} block = {}".format(i, j, block))
234
- non_24_segments += 1
235
-
236
- print(f"{non_24_segments} / {total_segments} do not have 2:4 structure.")
237
-
238
-
239
-def compress_quantized_24_weight(q_24, size_k, size_n, num_bits):
240
- assert q_24.shape == (size_k, size_n)
241
-
242
- # Remove zp to normalize over 0
243
- max_q_val = (1 << num_bits) - 1
244
- zp = (max_q_val + 1) // 2
245
- q_24_no_zp = q_24 - zp
246
-
247
- # Compress
248
- q_24_no_zp = q_24_no_zp.t().contiguous()
249
- q_24_no_zp_comp, meta = sparse_semi_structured_from_dense_cutlass(
250
- q_24_no_zp)
251
- q_24_no_zp_comp = q_24_no_zp_comp.t().contiguous()
252
-
253
- # Restore zp
254
- q_24_comp = q_24_no_zp_comp + zp
255
-
256
- # Resize meta to its actual shape (without moving any data)
257
- meta = meta.resize_(meta.shape[1] // 2, meta.shape[0] * 2)
258
-
259
- return q_24_comp, meta
260
-
261
-
262
-def marlin_24_quantize(
263
- w: torch.Tensor,
264
- num_bits: int,
265
- group_size: int,
266
-):
267
- size_k, size_n = w.shape
268
-
269
- # Normalize group_size
270
- if group_size == -1:
271
- group_size = size_k
272
- assert group_size <= size_k
273
-
274
- # Inject 2:4 sparsity
275
- w_24, mask_24 = inject_24(w, size_k, size_n)
276
-
277
- # Quantize
278
- w_24_ref, q_w_24, s, g_idx, rand_perm = quantize_weights(w_24,
279
- num_bits,
280
- group_size,
281
- act_order=False)
282
-
283
- # Compress quantized weight
284
- q_w_24_comp, meta = compress_quantized_24_weight(q_w_24, size_k, size_n,
285
- num_bits)
286
- size_k_comp = size_k // 2
287
-
288
- # Reformat to marlin
289
- marlin_24_q_w_comp = marlin_weights(q_w_24_comp, size_k_comp, size_n,
290
- num_bits, marlin_24_perm[num_bits])
291
- marlin_24_s = marlin_permute_scales(s, size_k, size_n, group_size,
292
- marlin_24_scale_perm[num_bits],
293
- marlin_24_scale_perm_single[num_bits])
294
-
295
- # Create result
296
- res_list = [w_24_ref, marlin_24_q_w_comp, meta, marlin_24_s]
297
- for i in range(len(res_list)):
298
- res_list[i] = res_list[i].to(w.device)
299
-
300
- return res_list
301
-
302
-
303
-def compute_max_diff(output, output_ref):
304
- return torch.mean(torch.abs(output - output_ref)) / torch.mean(
305
- torch.abs(output_ref))
306
-
307
-
308
-class MarlinWorkspace:
309
-
310
- def __init__(self, out_features, min_thread_n, max_parallel):
311
- assert (out_features % min_thread_n == 0), (
312
- "out_features = {} is undivisible by min_thread_n = {}".format(
313
- out_features, min_thread_n))
314
-
315
- max_workspace_size = ((out_features // min_thread_n) * max_parallel)
316
-
317
- self.scratch = torch.zeros(max_workspace_size,
318
- dtype=torch.int,
319
- device="cuda")
320
-
321
-
322
-def pack_fp8_to_int32(fp8_tensor: torch.Tensor) -> torch.Tensor:
323
- """
324
- Repack FP8 weights to gptq format (packed int32 elements)
325
- """
326
- assert fp8_tensor.dtype == torch.float8_e4m3fn
327
- assert fp8_tensor.shape[0] % 4 == 0
328
-
329
- # Reshape to prepare for packing
330
- reshaped = fp8_tensor.reshape(-1, 4, *fp8_tensor.shape[1:])
331
-
332
- # Convert fp8 to uint8 (byte) representation
333
- byte_tensor = reshaped.view(torch.uint8)
129
+# Newly generated tensors need to replace existing tensors that are
130
+# already registered as parameters by vLLM (and won't be freed)
131
+def replace_tensor(layer: torch.nn.Module, name: str,
132
+ new_t: torch.Tensor) -> None:
133
+ # It is important to use resize_() here since it ensures
134
+ # the same buffer is reused
135
+ getattr(layer, name).resize_(new_t.shape)
136
+ getattr(layer, name).copy_(new_t)
137
+ del new_t
138
+
139
+
140
+def apply_marlin_linear(input: torch.Tensor,
141
+ weight: torch.Tensor,
142
+ weight_scale: torch.Tensor,
143
+ g_idx: torch.Tensor,
144
+ g_idx_sort_indices: torch.Tensor,
145
+ workspace: torch.Tensor,
146
+ num_bits: int,
147
+ output_size_per_partition: int,
148
+ input_size_per_partition: int,
149
+ is_k_full: bool,
150
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
151
+ reshaped_x = input.reshape(-1, input.shape[-1])
152
+ out_shape = input.shape[:-1] + (output_size_per_partition, )
153
+
154
+ output = ops.gptq_marlin_gemm(reshaped_x,
155
+ weight,
156
+ weight_scale,
157
+ g_idx,
158
+ g_idx_sort_indices,
159
+ workspace,
160
+ num_bits,
161
+ size_m=reshaped_x.shape[0],
162
+ size_n=output_size_per_partition,
163
+ size_k=input_size_per_partition,
164
+ is_k_full=is_k_full)
165
335
- # Pack 4 uint8 values into one int32
336
- packed = (byte_tensor[:, 0].to(torch.int32) |
337
- (byte_tensor[:, 1].to(torch.int32) << 8) |
338
- (byte_tensor[:, 2].to(torch.int32) << 16) |
339
- (byte_tensor[:, 3].to(torch.int32) << 24))
166
+ if bias is not None:
167
+ output.add_(bias) # In-place add
168
341
- return packed.view(fp8_tensor.shape[0] // 4,
342
- *fp8_tensor.shape[1:]).contiguous()
169
+ return output.reshape(out_shape)
vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py
new
+109
@@ -0,0 +1,109 @@
1
+from typing import Optional
2
+
3
+import torch
4
+
5
+import vllm._custom_ops as ops
6
+from vllm.platforms import current_platform
7
+from vllm.utils import print_warning_once
8
+
9
+from .marlin_utils import marlin_make_workspace, marlin_permute_scales
10
+
11
+
12
+def is_fp8_marlin_supported():
13
+ capability = current_platform.get_device_capability()
14
+ return capability[0] >= 8
15
+
16
+
17
+def apply_fp8_marlin_linear(
18
+ input: torch.Tensor,
19
+ weight: torch.Tensor,
20
+ weight_scale: torch.Tensor,
21
+ workspace: torch.Tensor,
22
+ size_n: int,
23
+ size_k: int,
24
+ bias: Optional[torch.Tensor],
25
+) -> torch.Tensor:
26
+ # For GPUs that lack FP8 hardware support, we can leverage the
27
+ # Marlin kernel for fast weight-only FP8 quantization
28
+
29
+ reshaped_x = input.reshape(-1, input.shape[-1])
30
+ out_shape = input.shape[:-1] + (size_n, )
31
+
32
+ output = ops.fp8_marlin_gemm(
33
+ a=reshaped_x,
34
+ b_q_weight=weight,
35
+ b_scales=weight_scale,
36
+ workspace=workspace,
37
+ num_bits=8,
38
+ size_m=reshaped_x.shape[0],
39
+ size_n=size_n,
40
+ size_k=size_k,
41
+ )
42
+
43
+ if bias is not None:
44
+ output.add_(bias) # In-place add
45
+
46
+ return output.reshape(out_shape)
47
+
48
+
49
+def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
50
+ print_warning_once(
51
+ "Your GPU does not have native support for FP8 computation but "
52
+ "FP8 quantization is being used. Weight-only FP8 compression will "
53
+ "be used leveraging the Marlin kernel. This may degrade "
54
+ "performance for compute-heavy workloads.")
55
+
56
+ part_size_n = layer.output_size_per_partition
57
+ part_size_k = layer.input_size_per_partition
58
+
59
+ device = layer.weight.device
60
+
61
+ # WORKSPACE
62
+ layer.workspace = marlin_make_workspace(part_size_n, device)
63
+
64
+ # WEIGHT
65
+ # Repack weights to marlin format
66
+ marlin_qweight = ops.gptq_marlin_repack(b_q_weight=pack_fp8_to_int32(
67
+ layer.weight),
68
+ perm=torch.empty(0,
69
+ dtype=torch.int,
70
+ device=device),
71
+ size_k=part_size_k,
72
+ size_n=part_size_n,
73
+ num_bits=8)
74
+ layer.weight = torch.nn.Parameter(marlin_qweight, requires_grad=False)
75
+
76
+ # WEIGHT SCALES
77
+ # Currently Marlin doesn't support per-tensor scales, so we
78
+ # expand it to channelwise
79
+ scales = layer.weight_scale.repeat(1, part_size_n).to(
80
+ layer.orig_dtype).to(device)
81
+ # Permute scales
82
+ marlin_scales = marlin_permute_scales(s=scales,
83
+ size_k=part_size_k,
84
+ size_n=part_size_n,
85
+ group_size=-1)
86
+ layer.weight_scale = torch.nn.Parameter(marlin_scales, requires_grad=False)
87
+
88
+
89
+def pack_fp8_to_int32(fp8_tensor: torch.Tensor) -> torch.Tensor:
90
+ """
91
+ Repack FP8 weights to gptq format (packed int32 elements)
92
+ """
93
+ assert fp8_tensor.dtype == torch.float8_e4m3fn
94
+ assert fp8_tensor.shape[0] % 4 == 0
95
+
96
+ # Reshape to prepare for packing
97
+ reshaped = fp8_tensor.reshape(-1, 4, *fp8_tensor.shape[1:])
98
+
99
+ # Convert fp8 to uint8 (byte) representation
100
+ byte_tensor = reshaped.view(torch.uint8)
101
+
102
+ # Pack 4 uint8 values into one int32
103
+ packed = (byte_tensor[:, 0].to(torch.int32) |
104
+ (byte_tensor[:, 1].to(torch.int32) << 8) |
105
+ (byte_tensor[:, 2].to(torch.int32) << 16) |
106
+ (byte_tensor[:, 3].to(torch.int32) << 24))
107
+
108
+ return packed.view(fp8_tensor.shape[0] // 4,
109
+ *fp8_tensor.shape[1:]).contiguous()
vllm/model_executor/layers/quantization/utils/marlin_utils_test.py
new
+120
@@ -0,0 +1,120 @@
1
+"""Utility functions used for tests and benchmarks"""
2
+
3
+from typing import List
4
+
5
+import numpy
6
+import torch
7
+
8
+from .marlin_utils import GPTQ_MARLIN_TILE, marlin_permute_scales
9
+from .quant_utils import get_pack_factor, quantize_weights, sort_weights
10
+
11
+
12
+class MarlinWorkspace:
13
+
14
+ def __init__(self, out_features, min_thread_n, max_parallel):
15
+ assert (out_features % min_thread_n == 0), (
16
+ "out_features = {} is undivisible by min_thread_n = {}".format(
17
+ out_features, min_thread_n))
18
+
19
+ max_workspace_size = ((out_features // min_thread_n) * max_parallel)
20
+
21
+ self.scratch = torch.zeros(max_workspace_size,
22
+ dtype=torch.int,
23
+ device="cuda")
24
+
25
+
26
+def marlin_permute_weights(q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE):
27
+ assert q_w.shape == (size_k, size_n)
28
+ assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
29
+ assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
30
+
31
+ # Permute weights to 16x64 marlin tiles
32
+ q_w = q_w.reshape((size_k // tile, tile, size_n // tile, tile))
33
+ q_w = q_w.permute((0, 2, 1, 3))
34
+ q_w = q_w.reshape((size_k // tile, size_n * tile))
35
+
36
+ q_w = q_w.reshape((-1, perm.numel()))[:, perm].reshape(q_w.shape)
37
+
38
+ return q_w
39
+
40
+
41
+def marlin_weights(q_w, size_k, size_n, num_bits, perm):
42
+ # Permute
43
+ q_w = marlin_permute_weights(q_w, size_k, size_n, perm)
44
+
45
+ # Pack
46
+ pack_factor = get_pack_factor(num_bits)
47
+ orig_device = q_w.device
48
+
49
+ q_w = q_w.cpu().numpy().astype(numpy.uint32)
50
+
51
+ q_packed = numpy.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
52
+ dtype=numpy.uint32)
53
+ for i in range(pack_factor):
54
+ q_packed |= q_w[:, i::pack_factor] << num_bits * i
55
+
56
+ q_packed = torch.from_numpy(q_packed.astype(numpy.int32)).to(orig_device)
57
+
58
+ return q_packed
59
+
60
+
61
+def get_weight_perm(num_bits: int):
62
+ perm_list: List[int] = []
63
+ for i in range(32):
64
+ perm1: List[int] = []
65
+ col = i // 4
66
+ for block in [0, 1]:
67
+ for row in [
68
+ 2 * (i % 4),
69
+ 2 * (i % 4) + 1,
70
+ 2 * (i % 4 + 4),
71
+ 2 * (i % 4 + 4) + 1,
72
+ ]:
73
+ perm1.append(16 * row + col + 8 * block)
74
+ for j in range(4):
75
+ perm_list.extend([p + 256 * j for p in perm1])
76
+
77
+ perm = numpy.array(perm_list)
78
+
79
+ if num_bits == 4:
80
+ interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
81
+ elif num_bits == 8:
82
+ interleave = numpy.array([0, 2, 1, 3])
83
+ else:
84
+ raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
85
+
86
+ perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
87
+ perm = torch.from_numpy(perm)
88
+ return perm
89
+
90
+
91
+def marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int,
92
+ act_order: bool):
93
+ size_k, size_n = w.shape
94
+
95
+ # Normalize group_size
96
+ if group_size == -1:
97
+ group_size = size_k
98
+ assert group_size <= size_k
99
+
100
+ # Quantize (and apply act_order if provided)
101
+ w_ref, q_w, s, g_idx, rand_perm = quantize_weights(w, num_bits, group_size,
102
+ act_order)
103
+
104
+ # For act_order, sort the "weights" and "g_idx" so that group ids are
105
+ # increasing
106
+ sort_indices = torch.empty(0, dtype=torch.int, device=w.device)
107
+ if act_order:
108
+ q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
109
+
110
+ # Reformat to marlin
111
+ weight_perm = get_weight_perm(num_bits)
112
+ marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
113
+ marlin_s = marlin_permute_scales(s, size_k, size_n, group_size)
114
+
115
+ # Create result
116
+ res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm]
117
+ for i in range(len(res_list)):
118
+ res_list[i] = res_list[i].to(w.device)
119
+
120
+ return res_list
vllm/model_executor/layers/quantization/utils/marlin_utils_test_24.py
renamed
+160
-3
@@ -1,9 +1,14 @@
1
-#
2
-# Modified by Roberto Lopez Castro (roberto.lopez.castro@udc.es).
3
-#
1
+"""Utility functions used for tests and benchmarks"""
2
3
+import random
4
+from typing import List
5
+
6
+import numpy
7
import torch
8
9
+from .marlin_utils_test import marlin_weights
10
+from .quant_utils import quantize_weights
11
+
12
13
# This is PyTorch implementation of main part of reorder_meta()
14
# function, from tools/util/include/cutlass/util/host_reorder.h file
@@ -306,3 +311,155 @@ def mask_creator(tensor):
311
mask = w_b.scatter_(dim=1, index=index, value=0).reshape(tensor.shape)
312
313
return mask
314
+
315
+
316
+def inject_24(w, size_k, size_n):
317
+ assert w.shape == (size_k, size_n)
318
+
319
+ mask = mask_creator(w.t()).t().cuda().bool()
320
+
321
+ return (mask * w).contiguous(), mask.contiguous()
322
+
323
+
324
+def check_24(w, num_rows_to_sample=50, _verbose=False):
325
+ BLOCK_SIZE = 4
326
+ MAX_NON_ZEROS = 2
327
+
328
+ w = w.t().contiguous()
329
+
330
+ print("check_24: w.shape = {}".format(w.shape))
331
+
332
+ num_rows, num_cols = w.shape
333
+ sampled_row_idxs = random.choices(range(num_rows), k=num_rows_to_sample)
334
+ if _verbose:
335
+ print(f"Sampled row idxs = {sampled_row_idxs}")
336
+
337
+ total_segments = 0
338
+ non_24_segments = 0
339
+ for i in sampled_row_idxs:
340
+ for j in range(0, num_cols - BLOCK_SIZE, BLOCK_SIZE):
341
+ total_segments += 1
342
+ block = w[i, j:j + BLOCK_SIZE]
343
+ num_nonzero = torch.count_nonzero(block)
344
+ if num_nonzero > MAX_NON_ZEROS:
345
+ print("i = {} j = {} block = {}".format(i, j, block))
346
+ non_24_segments += 1
347
+
348
+ print(f"{non_24_segments} / {total_segments} do not have 2:4 structure.")
349
+
350
+
351
+def compress_quantized_24_weight(q_24, size_k, size_n, num_bits):
352
+ assert q_24.shape == (size_k, size_n)
353
+
354
+ # Remove zp to normalize over 0
355
+ max_q_val = (1 << num_bits) - 1
356
+ zp = (max_q_val + 1) // 2
357
+ q_24_no_zp = q_24 - zp
358
+
359
+ # Compress
360
+ q_24_no_zp = q_24_no_zp.t().contiguous()
361
+ q_24_no_zp_comp, meta = sparse_semi_structured_from_dense_cutlass(
362
+ q_24_no_zp)
363
+ q_24_no_zp_comp = q_24_no_zp_comp.t().contiguous()
364
+
365
+ # Restore zp
366
+ q_24_comp = q_24_no_zp_comp + zp
367
+
368
+ # Resize meta to its actual shape (without moving any data)
369
+ meta = meta.resize_(meta.shape[1] // 2, meta.shape[0] * 2)
370
+
371
+ return q_24_comp, meta
372
+
373
+
374
+def get_scale_perms_24():
375
+ scale_perm: List[int] = []
376
+ for i in range(8):
377
+ scale_perm.extend([i * 8 + j for j in [0, 4, 1, 5, 2, 6, 3, 7]])
378
+ scale_perm_single: List[int] = []
379
+ for i in range(8):
380
+ scale_perm_single.extend([8 * i + j for j in [0, 1, 2, 3, 4, 5, 6, 7]])
381
+ return scale_perm, scale_perm_single
382
+
383
+
384
+def get_weight_perm_24(num_bits: int):
385
+ perm_list: List[int] = []
386
+ for i in range(32):
387
+ perm1: List[int] = []
388
+ col = i // 4
389
+ col_o = col // 2
390
+ for block in [0, 1]:
391
+ for row in [
392
+ 2 * (i % 4),
393
+ 2 * (i % 4) + 1,
394
+ 2 * (i % 4 + 4),
395
+ 2 * (i % 4 + 4) + 1,
396
+ ]:
397
+ perm1.append(16 * row + col_o * 256 + 8 * (col % 2) +
398
+ 4 * block)
399
+ for j in range(4):
400
+ perm_list.extend([p + 1 * j for p in perm1])
401
+ perm = numpy.array(perm_list)
402
+
403
+ if num_bits == 4:
404
+ interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
405
+ elif num_bits == 8:
406
+ interleave = numpy.array([0, 2, 1, 3])
407
+ else:
408
+ raise ValueError("num_bits must be 4 or 8, got {}".format(num_bits))
409
+
410
+ perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
411
+ perm = torch.from_numpy(perm)
412
+ return perm
413
+
414
+
415
+def marlin_permute_scales_24(s: torch.Tensor, size_k: int, size_n: int,
416
+ group_size: int) -> torch.Tensor:
417
+
418
+ scale_perm, scale_perm_single = get_scale_perms_24()
419
+ if group_size < size_k and group_size != -1:
420
+ s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
421
+ else:
422
+ s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single]
423
+ s = s.reshape((-1, size_n)).contiguous()
424
+
425
+ return s
426
+
427
+
428
+def marlin_24_quantize(
429
+ w: torch.Tensor,
430
+ num_bits: int,
431
+ group_size: int,
432
+):
433
+ size_k, size_n = w.shape
434
+
435
+ # Normalize group_size
436
+ if group_size == -1:
437
+ group_size = size_k
438
+ assert group_size <= size_k
439
+
440
+ # Inject 2:4 sparsity
441
+ w_24, mask_24 = inject_24(w, size_k, size_n)
442
+
443
+ # Quantize
444
+ w_24_ref, q_w_24, s, g_idx, rand_perm = quantize_weights(w_24,
445
+ num_bits,
446
+ group_size,
447
+ act_order=False)
448
+
449
+ # Compress quantized weight
450
+ q_w_24_comp, meta = compress_quantized_24_weight(q_w_24, size_k, size_n,
451
+ num_bits)
452
+ size_k_comp = size_k // 2
453
+
454
+ # Reformat to marlin
455
+ weight_perm = get_weight_perm_24(num_bits)
456
+ marlin_24_q_w_comp = marlin_weights(q_w_24_comp, size_k_comp, size_n,
457
+ num_bits, weight_perm)
458
+ marlin_24_s = marlin_permute_scales_24(s, size_k, size_n, group_size)
459
+
460
+ # Create result
461
+ res_list = [w_24_ref, marlin_24_q_w_comp, meta, marlin_24_s]
462
+ for i in range(len(res_list)):
463
+ res_list[i] = res_list[i].to(w.device)
464
+
465
+ return res_list