@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b675069d74
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ Misc ] Refactor Marlin Python Utilities (#6082)

Co-authored-by: Robert Shaw <rshaw@neuralmagic.com>

Robert Shaw committed Jul 11, 2024 at 11:40 UTC b675069d7486129dbed7847f420b7a927691f16b
12 files changed +704 -742
benchmarks/kernels/benchmark_marlin.py
+6 -4
@@ -5,14 +5,16 @@ import torch.utils.benchmark as benchmark
5 from benchmark_shapes import WEIGHT_SHAPES
6
7 from vllm import _custom_ops as ops
8 -from vllm.model_executor.layers.quantization.gptq_marlin import (
9 - GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
10 - GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
8 from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
9 GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
10 GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
11 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15 - MarlinWorkspace, marlin_24_quantize, marlin_quantize)
12 + GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
13 + GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
14 +from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
15 + MarlinWorkspace, marlin_quantize)
16 +from vllm.model_executor.layers.quantization.utils.marlin_utils_test_24 import (
17 + marlin_24_quantize)
18 from vllm.model_executor.layers.quantization.utils.quant_utils import (
19 gptq_pack, quantize_weights, sort_weights)
20 from vllm.utils import FlexibleArgumentParser
tests/kernels/test_marlin_gemm.py
+25 -21
@@ -5,19 +5,21 @@ Run `pytest tests/kernels/marlin/test_marlin_gemm.py`.
5 import pytest
6 import torch
7
8 +from tests.quantization.utils import is_quant_method_supported
9 from vllm import _custom_ops as ops
9 -from vllm.model_executor.layers.quantization.gptq_marlin import (
10 - GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
11 - GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS,
12 - marlin_permute_scales)
10 from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
11 GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
12 GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
16 -from vllm.model_executor.layers.quantization.utils.marlin_perms import (
17 - marlin_perm)
13 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
19 - MarlinWorkspace, compute_max_diff, is_marlin_supported, marlin_24_quantize,
20 - marlin_quantize, marlin_weights, pack_fp8_to_int32)
14 + GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
15 + GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS,
16 + marlin_permute_scales)
17 +from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
18 + pack_fp8_to_int32)
19 +from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
20 + MarlinWorkspace, get_weight_perm, marlin_quantize, marlin_weights)
21 +from vllm.model_executor.layers.quantization.utils.marlin_utils_test_24 import (
22 + marlin_24_quantize)
23 from vllm.model_executor.layers.quantization.utils.quant_utils import (
24 gptq_pack, quantize_weights, sort_weights)
25
@@ -42,11 +44,16 @@ MNK_FACTORS = [
44 DTYPES = [torch.float16, torch.bfloat16]
45
46
47 +def compute_max_diff(output, output_ref):
48 + return torch.mean(torch.abs(output - output_ref)) / torch.mean(
49 + torch.abs(output_ref))
50 +
51 +
52 def rand_data(shape, dtype=torch.float16):
53 return torch.randn(shape, dtype=dtype, device="cuda")
54
55
49 -@pytest.mark.skipif(not is_marlin_supported(),
56 +@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
57 reason="Marlin is not supported on this GPU type.")
58 @pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
59 @pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
@@ -93,8 +100,8 @@ def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
100 q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
101
102 # Pack to Marlin format
96 - marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits,
97 - marlin_perm[num_bits])
103 + weight_perm = get_weight_perm(num_bits)
104 + marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
105
106 # Run Marlin repack GPU kernel
107 marlin_q_w_2 = ops.gptq_marlin_repack(
@@ -109,7 +116,7 @@ def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
116 assert torch.allclose(marlin_q_w_1, marlin_q_w_2)
117
118
112 -@pytest.mark.skipif(not is_marlin_supported(),
119 +@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
120 reason="Marlin is not supported on this GPU type.")
121 @pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
122 @pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
@@ -174,7 +181,7 @@ def test_marlin_gemm(
181 assert max_diff < 0.04
182
183
177 -@pytest.mark.skipif(not is_marlin_supported(),
184 +@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
185 reason="Marlin is not supported on this GPU type.")
186 @pytest.mark.parametrize("k_chunk", MARLIN_24_K_CHUNKS)
187 @pytest.mark.parametrize("n_chunk", MARLIN_24_N_CHUNKS)
@@ -222,7 +229,7 @@ def test_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size, mnk_factors):
229 assert max_diff < 0.04
230
231
225 -@pytest.mark.skipif(not is_marlin_supported(),
232 +@pytest.mark.skipif(not is_quant_method_supported("fp8"),
233 reason="Marlin is not supported on this GPU type.")
234 @pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
235 @pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
@@ -268,13 +275,10 @@ def test_fp8_marlin_gemm(
275 # expand it to channelwise
276 scales = weight_scale.repeat(1, size_n).to(a_input.dtype).to("cuda")
277 # Permute scales
271 - marlin_scales = marlin_permute_scales(
272 - s=scales,
273 - size_k=size_k,
274 - size_n=size_n,
275 - group_size=-1,
276 - num_bits=8,
277 - )
278 + marlin_scales = marlin_permute_scales(s=scales,
279 + size_k=size_k,
280 + size_n=size_n,
281 + group_size=-1)
282
283 workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
284 GPTQ_MARLIN_MAX_PARALLEL)
tests/quantization/test_compressed_tensors.py
+14 -9
@@ -6,7 +6,6 @@ Run `pytest tests/quantization/test_compressed_tensors.py`.
6 import pytest
7 import torch
8
9 -from vllm import SamplingParams
9 from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
10 CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
11 CompressedTensorsW8A8Fp8, CompressedTensorsW8A8Int8,
@@ -57,12 +56,14 @@ def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
56 assert qkv_proj.weight_scale.dtype is torch.float32
57 assert qkv_proj.input_scale.dtype is torch.float32
58
59 + output = llm.generate_greedy("Hello my name is", max_tokens=20)
60 + assert output
61 +
62
63 def test_compressed_tensors_no_enforce_eager(vllm_runner):
64 model_path = "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change"
65 with vllm_runner(model_path) as llm:
64 - sampling_params = SamplingParams()
65 - output = llm.generate("Hello world!", sampling_params=sampling_params)
66 + output = llm.generate_greedy("Hello my name is", max_tokens=20)
67 assert output
68
69
@@ -84,13 +85,16 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
85 assert qkv_proj.scheme.strategy == strategy
86 assert qkv_proj.weight.dtype is torch.int8
87
88 + output = llm.generate_greedy("Hello my name is", max_tokens=20)
89 + assert output
90 +
91
92 @pytest.mark.parametrize(
93 "wNa16_args",
94 [("nm-testing/tinyllama-oneshot-w4a16-channel-v2", "channel", None, 8),
95 ("nm-testing/tinyllama-oneshot-w4a16-group128-v2", "group", 128, 8),
96 ("nm-testing/tinyllama-oneshot-w8a16-per-channel", "channel", None, 4)])
93 -def test_compressed_tensors_w4a16(vllm_runner, wNa16_args):
97 +def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
98 model, strategy, group, pack_factor = wNa16_args
99 with vllm_runner(model) as llm:
100 model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
@@ -101,12 +105,15 @@ def test_compressed_tensors_w4a16(vllm_runner, wNa16_args):
105 assert isinstance(qkv_proj.scheme, CompressedTensorsWNA16)
106
107 assert qkv_proj.scheme.strategy == strategy
104 - assert qkv_proj.scheme.group_size == group
108 + assert qkv_proj.scheme.group_size == (-1 if group is None else group)
109
110 assert qkv_proj.weight_packed.dtype is torch.int32
111 assert qkv_proj.weight_scale.dtype is torch.float16
112 assert qkv_proj.weight_packed.pack_factor == pack_factor
113
114 + output = llm.generate_greedy("Hello my name is", max_tokens=20)
115 + assert output
116 +
117
118 def test_compressed_tensors_w4a16_marlin24(vllm_runner):
119 model_path = "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t"
@@ -120,8 +127,7 @@ def test_compressed_tensors_w4a16_marlin24(vllm_runner):
127 assert isinstance(qkv_proj.scheme, CompressedTensorsW4A16Sparse24)
128 assert qkv_proj.weight_packed.dtype is torch.int32
129
123 - sampling_params = SamplingParams()
124 - output = llm.generate("Hello world!", sampling_params=sampling_params)
130 + output = llm.generate_greedy("Hello my name is", max_tokens=20)
131 assert output
132
133
@@ -142,6 +148,5 @@ def test_compressed_tensors_fp8(vllm_runner):
148 assert len(qkv_proj.input_scale.shape) == 0
149 assert len(qkv_proj.weight_scale.shape) == 0
150
145 - sampling_params = SamplingParams()
146 - output = llm.generate("Hello world!", sampling_params=sampling_params)
151 + output = llm.generate_greedy("Hello my name is", max_tokens=20)
152 assert output
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+69 -82
@@ -6,9 +6,10 @@ from torch.nn import Parameter
6 from vllm import _custom_ops as ops
7 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 CompressedTensorsScheme)
9 -from vllm.model_executor.layers.quantization.gptq_marlin import (
10 - GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, GPTQMarlinState,
11 - marlin_permute_scales)
9 +from vllm.model_executor.layers.quantization.utils.marlin_utils import (
10 + apply_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace,
11 + marlin_permute_scales, replace_tensor, verify_marlin_supported,
12 + verify_marlin_supports_shape)
13 from vllm.model_executor.utils import set_weight_attrs
14
15 __all__ = ["CompressedTensorsWNA16"]
@@ -22,29 +23,40 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
23 num_bits: int,
24 group_size: Optional[int] = None):
25 self.num_bits = num_bits
26 + self.pack_factor = 32 // self.num_bits
27 self.strategy = strategy
26 - self.group_size = group_size
28
28 - if self.strategy == "group" and self.group_size is None:
29 - raise ValueError(
30 - "group_size must be given when using strategy group")
29 + self.group_size: int
30 + if group_size is None:
31 + if self.strategy != "channel":
32 + raise ValueError(
33 + "Marlin kernels require group quantization or "
34 + "channelwise quantization, but found no group "
35 + "size and strategy is not channelwise.")
36 + self.group_size = -1
37 + else:
38 + self.group_size = group_size
39
32 - def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
33 - pass
40 + # Verify supported on platform.
41 + verify_marlin_supported(num_bits=self.num_bits,
42 + group_size=self.group_size,
43 + is_sym=True)
44
45 def create_weights(self, layer: torch.nn.Module, input_size: int,
46 output_partition_sizes: List[int],
47 input_size_per_partition: int,
48 params_dtype: torch.dtype, weight_loader: Callable,
49 **kwargs):
40 -
41 - pack_factor = 32 // self.num_bits
50 output_size_per_partition = sum(output_partition_sizes)
51
44 - if self.group_size is not None:
45 - group_size = self.group_size
46 - else:
47 - group_size = input_size
52 + # If group_size is -1, we are in channelwise case.
53 + group_size = input_size if self.group_size == -1 else self.group_size
54 +
55 + verify_marlin_supports_shape(
56 + output_size_per_partition=output_size_per_partition,
57 + input_size_per_partition=input_size_per_partition,
58 + input_size=input_size,
59 + group_size=group_size)
60
61 weight_scale_dim = None
62 scales_and_zp_size = input_size // group_size
@@ -57,7 +69,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
69 weight = Parameter(
70 torch.empty(
71 output_size_per_partition,
60 - input_size_per_partition // pack_factor,
72 + input_size_per_partition // self.pack_factor,
73 dtype=torch.int32,
74 ),
75 requires_grad=False,
@@ -68,7 +80,7 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
80 "input_dim": 1,
81 "output_dim": 0,
82 "packed_dim": 1,
71 - "pack_factor": pack_factor,
83 + "pack_factor": self.pack_factor,
84 "weight_loader": weight_loader
85 })
86 layer.register_parameter("weight_packed", weight)
@@ -103,73 +115,48 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
115
116 layer.input_size_per_partition = input_size_per_partition
117 layer.output_size_per_partition = output_size_per_partition
106 -
118 layer.input_size = input_size
108 - layer.marlin_state = GPTQMarlinState.REPACK
109 - layer.is_k_full = True
119 layer.group_size = group_size
120
112 - max_workspace_size = (
113 - output_size_per_partition //
114 - GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL
115 -
116 - workspace = torch.zeros(max_workspace_size,
117 - dtype=torch.int,
118 - requires_grad=False)
119 - layer.workspace = workspace
121 + # Checkpoints are serialized in compressed-tensors format, which is
122 + # different from marlin format. Handle repacking here.
123 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
124 + device = layer.weight_packed.device
125 +
126 + # Allocate marlin workspace.
127 + layer.workspace = marlin_make_workspace(
128 + layer.output_size_per_partition, device)
129 +
130 + # Act-order not supported in compressed-tensors yet, so set to empty.
131 + layer.g_idx = marlin_make_empty_g_idx(device)
132 + layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
133 +
134 + # Repack weights from compressed-tensors format to marlin format.
135 + marlin_qweight = ops.gptq_marlin_repack(
136 + layer.weight_packed.t().contiguous(),
137 + perm=layer.g_idx_sort_indices,
138 + size_k=layer.input_size_per_partition,
139 + size_n=layer.output_size_per_partition,
140 + num_bits=self.num_bits)
141 + replace_tensor(layer, "weight_packed", marlin_qweight)
142 +
143 + # Permute scales from compressed-tensors format to marlin format.
144 + marlin_scales = marlin_permute_scales(
145 + layer.weight_scale.squeeze().t().contiguous(),
146 + size_k=layer.input_size_per_partition,
147 + size_n=layer.output_size_per_partition,
148 + group_size=layer.group_size)
149 + replace_tensor(layer, "weight_scale", marlin_scales)
150
151 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor):
122 - reshaped_x = x.reshape(-1, x.shape[-1])
123 -
124 - size_m = reshaped_x.shape[0]
125 - part_size_n = layer.output_size_per_partition
126 - part_size_k = layer.input_size_per_partition
127 -
128 - out_shape = x.shape[:-1] + (part_size_n, )
129 -
130 - if layer.marlin_state == GPTQMarlinState.REPACK:
131 - layer.marlin_state = GPTQMarlinState.READY
132 -
133 - # Newly generated tensors need to replace existing tensors that are
134 - # already registered as parameters by vLLM (and won't be freed)
135 - def replace_tensor(name, new_t):
136 - # It is important to use resize_() here since it ensures
137 - # the same buffer is reused
138 - getattr(layer, name).resize_(new_t.shape)
139 - getattr(layer, name).copy_(new_t)
140 - del new_t
141 -
142 - cur_device = layer.weight_packed.device
143 -
144 - # Reset g_idx related tensors
145 - layer.g_idx = Parameter(torch.empty(0,
146 - dtype=torch.int,
147 - device=cur_device),
148 - requires_grad=False)
149 - layer.g_idx_sort_indices = Parameter(torch.empty(
150 - 0, dtype=torch.int, device=cur_device),
151 - requires_grad=False)
152 -
153 - # Repack weights
154 - marlin_qweight = ops.gptq_marlin_repack(
155 - layer.weight_packed.t().contiguous(), layer.g_idx_sort_indices,
156 - part_size_k, part_size_n, self.num_bits)
157 -
158 - replace_tensor("weight_packed", marlin_qweight)
159 -
160 - # Permute scales
161 - scales_size_k = part_size_k
162 - scales_size_n = part_size_n
163 -
164 - marlin_scales = marlin_permute_scales(
165 - layer.weight_scale.squeeze().t().contiguous(), scales_size_k,
166 - scales_size_n, layer.group_size, self.num_bits)
167 - replace_tensor("weight_scale", marlin_scales)
168 -
169 - output = ops.gptq_marlin_gemm(reshaped_x, layer.weight_packed,
170 - layer.weight_scale, layer.g_idx,
171 - layer.g_idx_sort_indices,
172 - layer.workspace, self.num_bits, size_m,
173 - part_size_n, part_size_k,
174 - layer.is_k_full)
175 - return output.reshape(out_shape)
152 + return apply_marlin_linear(
153 + input=x,
154 + weight=layer.weight_packed,
155 + weight_scale=layer.weight_scale,
156 + g_idx=layer.g_idx,
157 + g_idx_sort_indices=layer.g_idx_sort_indices,
158 + workspace=layer.workspace,
159 + num_bits=self.num_bits,
160 + output_size_per_partition=layer.output_size_per_partition,
161 + input_size_per_partition=layer.input_size_per_partition,
162 + is_k_full=True)
vllm/model_executor/layers/quantization/fp8.py
+1 -1
@@ -11,7 +11,7 @@ from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
11 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig, QuantizeMethodBase)
14 -from vllm.model_executor.layers.quantization.utils.marlin_utils import (
14 +from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
15 apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin)
16 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
17 all_close_1d, apply_fp8_linear, create_per_tensor_scale_param,
vllm/model_executor/layers/quantization/gptq_marlin.py
+67 -196
@@ -1,5 +1,3 @@
1 -import enum
2 -from enum import Enum
1 from typing import Any, Dict, List, Optional
2
3 import torch
@@ -12,46 +10,14 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10 from vllm.model_executor.layers.quantization.base_config import (
11 QuantizationConfig)
12 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15 - GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_K,
16 - GPTQ_MARLIN_MIN_THREAD_N, GPTQ_MARLIN_SUPPORTED_GROUP_SIZES,
17 - GPTQ_MARLIN_SUPPORTED_NUM_BITS, GPTQ_MARLIN_SUPPORTED_SYM,
18 - GPTQ_MARLIN_TILE)
13 + check_marlin_supported, marlin_make_empty_g_idx, marlin_make_workspace,
14 + marlin_permute_scales, marlin_sort_g_idx, replace_tensor,
15 + verify_marlin_supported, verify_marlin_supports_shape)
16 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
20 -from vllm.platforms import current_platform
17
18 logger = init_logger(__name__)
19
20
25 -# Permutations for Marlin scale shuffling
26 -def get_scale_perms(num_bits: int):
27 - scale_perm: List[int] = []
28 - for i in range(8):
29 - scale_perm.extend([i + 8 * j for j in range(8)])
30 - scale_perm_single: List[int] = []
31 - for i in range(4):
32 - scale_perm_single.extend(
33 - [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
34 - return scale_perm, scale_perm_single
35 -
36 -
37 -def get_pack_factor(num_bits: int):
38 - assert (num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS
39 - ), f"Unsupported num_bits = {num_bits}"
40 - return 32 // num_bits
41 -
42 -
43 -def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
44 - group_size: int, num_bits: int):
45 - scale_perm, scale_perm_single = get_scale_perms(num_bits)
46 - if group_size < size_k and group_size != -1:
47 - s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
48 - else:
49 - s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single]
50 - s = s.reshape((-1, size_n)).contiguous()
51 -
52 - return s
53 -
54 -
21 class GPTQMarlinConfig(QuantizationConfig):
22 """Config class for GPTQ Marlin"""
23
@@ -63,33 +29,16 @@ class GPTQMarlinConfig(QuantizationConfig):
29 desc_act = False
30
31 self.weight_bits = weight_bits
32 + self.pack_factor = 32 // self.weight_bits # packed into int32
33 self.group_size = group_size
34 self.desc_act = desc_act
35 self.is_sym = is_sym
36 self.lm_head_quantized = lm_head_quantized
37
71 - # Verify
72 - if self.weight_bits not in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
73 - raise ValueError(
74 - f"Marlin does not support weight_bits = {self.weight_bits}. "
75 - f"Only weight_bits = {GPTQ_MARLIN_SUPPORTED_NUM_BITS} "
76 - "are supported.")
77 - if self.group_size not in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES:
78 - raise ValueError(
79 - f"Marlin does not support group_size = {self.group_size}. "
80 - f"Only group_sizes = {GPTQ_MARLIN_SUPPORTED_GROUP_SIZES} "
81 - "are supported.")
82 - if self.is_sym not in GPTQ_MARLIN_SUPPORTED_SYM:
83 - raise ValueError(
84 - f"Marlin does not support is_sym = {self.is_sym}. "
85 - f"Only sym = {GPTQ_MARLIN_SUPPORTED_SYM} are supported.")
86 -
87 - # Init
88 - self.pack_factor = get_pack_factor(weight_bits)
89 - self.tile_size = GPTQ_MARLIN_TILE
90 - self.min_thread_n = GPTQ_MARLIN_MIN_THREAD_N
91 - self.min_thread_k = GPTQ_MARLIN_MIN_THREAD_K
92 - self.max_parallel = GPTQ_MARLIN_MAX_PARALLEL
38 + # Verify supported on platform.
39 + verify_marlin_supported(num_bits=self.weight_bits,
40 + group_size=self.group_size,
41 + is_sym=self.is_sym)
42
43 def __repr__(self) -> str:
44 return (f"GPTQMarlinConfig(weight_bits={self.weight_bits}, "
@@ -168,21 +117,10 @@ class GPTQMarlinConfig(QuantizationConfig):
117 or desc_act is None):
118 return False
119
171 - # If the capability of the device is too low, cannot convert.
172 - major, minor = current_platform.get_device_capability()
173 - device_capability = major * 10 + minor
174 - if device_capability < cls.get_min_capability():
175 - return False
176 -
177 - # Otherwise, can convert if model satisfies marlin constraints.
178 - return (num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS
179 - and group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES
180 - and sym in GPTQ_MARLIN_SUPPORTED_SYM)
181 -
182 -
183 -class GPTQMarlinState(Enum):
184 - REPACK = enum.auto()
185 - READY = enum.auto()
120 + return check_marlin_supported(num_bits=num_bits,
121 + group_size=group_size,
122 + is_sym=sym,
123 + min_capability=cls.get_min_capability())
124
125
126 class GPTQMarlinLinearMethod(LinearMethodBase):
@@ -206,6 +144,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
144 **extra_weight_attrs,
145 ) -> None:
146 del output_size
147 + output_size_per_partition = sum(output_partition_sizes)
148
149 # Normalize group_size
150 if self.quant_config.group_size != -1:
@@ -213,31 +152,11 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
152 else:
153 group_size = input_size
154
216 - # Validate dtype
217 - if params_dtype not in [torch.float16, torch.bfloat16]:
218 - raise ValueError(f"The params dtype must be float16 "
219 - f"or bfloat16, but got {params_dtype}")
220 -
221 - # Validate output_size_per_partition
222 - output_size_per_partition = sum(output_partition_sizes)
223 - if output_size_per_partition % self.quant_config.min_thread_n != 0:
224 - raise ValueError(
225 - f"Weight output_size_per_partition = "
226 - f"{output_size_per_partition} is not divisible by "
227 - f" min_thread_n = {self.quant_config.min_thread_n}.")
228 -
229 - # Validate input_size_per_partition
230 - if input_size_per_partition % self.quant_config.min_thread_k != 0:
231 - raise ValueError(
232 - f"Weight input_size_per_partition = "
233 - f"{input_size_per_partition} is not divisible "
234 - f"by min_thread_k = {self.quant_config.min_thread_k}.")
235 -
236 - if (group_size < input_size
237 - and input_size_per_partition % group_size != 0):
238 - raise ValueError(
239 - f"Weight input_size_per_partition = {input_size_per_partition}"
240 - f" is not divisible by group_size = {group_size}.")
155 + verify_marlin_supports_shape(
156 + output_size_per_partition=output_size_per_partition,
157 + input_size_per_partition=input_size_per_partition,
158 + input_size=input_size,
159 + group_size=group_size)
160
161 # Detect sharding of scales/zp
162
@@ -303,11 +222,6 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
222 },
223 )
224
306 - g_idx_sort_indices = torch.empty(
307 - g_idx.shape,
308 - dtype=torch.int32,
309 - )
310 -
225 # Scales
226 scales = Parameter(
227 torch.empty(
@@ -347,25 +261,50 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
261 },
262 )
263
350 - # Allocate marlin workspace
351 - max_workspace_size = (
352 - output_size_per_partition //
353 - self.quant_config.min_thread_n) * self.quant_config.max_parallel
354 - workspace = torch.zeros(max_workspace_size,
355 - dtype=torch.int,
356 - requires_grad=False)
357 -
264 layer.register_parameter("qweight", qweight)
265 layer.register_parameter("g_idx", g_idx)
266 layer.register_parameter("scales", scales)
267 layer.register_parameter("qzeros", qzeros)
362 - layer.g_idx_sort_indices = g_idx_sort_indices
363 - layer.workspace = workspace
268 layer.input_size_per_partition = input_size_per_partition
269 layer.output_size_per_partition = output_size_per_partition
270 layer.input_size = input_size
271 layer.is_k_full = is_k_full
368 - layer.marlin_state = GPTQMarlinState.REPACK
272 +
273 + # Checkpoints are serialized in AutoGPTQ format, which is different from the
274 + # marlin format. This function is called after the weights are loaded.
275 + # Here, we handle the repacking, including the activation reordering case.
276 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
277 + device = layer.qweight.device
278 + # Allocate marlin workspace
279 + layer.workspace = marlin_make_workspace(
280 + layer.output_size_per_partition, device)
281 +
282 + # Handle sorting for activation reordering if needed.
283 + if self.quant_config.desc_act:
284 + g_idx, g_idx_sort_indices = marlin_sort_g_idx(layer.g_idx)
285 + layer.g_idx_sort_indices = g_idx_sort_indices
286 + replace_tensor(layer, "g_idx", g_idx)
287 + else:
288 + layer.g_idx = marlin_make_empty_g_idx(device)
289 + layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
290 +
291 + # Repack weights from autogptq format to marlin format.
292 + marlin_qweight = ops.gptq_marlin_repack(
293 + layer.qweight,
294 + perm=layer.g_idx_sort_indices,
295 + size_k=layer.input_size_per_partition,
296 + size_n=layer.output_size_per_partition,
297 + num_bits=self.quant_config.weight_bits)
298 + replace_tensor(layer, "qweight", marlin_qweight)
299 +
300 + # Permute scales from autogptq format to marlin format.
301 + marlin_scales = marlin_permute_scales(
302 + layer.scales,
303 + size_k=(layer.input_size if self.quant_config.desc_act else
304 + layer.input_size_per_partition),
305 + size_n=layer.output_size_per_partition,
306 + group_size=self.quant_config.group_size)
307 + replace_tensor(layer, "scales", marlin_scales)
308
309 def apply(
310 self,
@@ -374,87 +313,19 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
313 bias: Optional[torch.Tensor] = None,
314 ) -> torch.Tensor:
315 reshaped_x = x.reshape(-1, x.shape[-1])
377 -
378 - size_m = reshaped_x.shape[0]
379 - part_size_n = layer.output_size_per_partition
380 - part_size_k = layer.input_size_per_partition
381 - full_size_k = layer.input_size
382 -
383 - out_shape = x.shape[:-1] + (part_size_n, )
384 -
385 - if layer.marlin_state == GPTQMarlinState.REPACK:
386 - layer.marlin_state = GPTQMarlinState.READY
387 -
388 - # Newly generated tensors need to replace existing tensors that are
389 - # already registered as parameters by vLLM (and won't be freed)
390 - def replace_tensor(name, new_t):
391 - # It is important to use resize_() here since it ensures
392 - # the same buffer is reused
393 - getattr(layer, name).resize_(new_t.shape)
394 - getattr(layer, name).copy_(new_t)
395 - del new_t
396 -
397 - cur_device = layer.qweight.device
398 -
399 - # Process act_order
400 - if self.quant_config.desc_act:
401 - # Get sorting based on g_idx
402 - g_idx_sort_indices = torch.argsort(layer.g_idx).to(torch.int)
403 -
404 - sorted_g_idx = layer.g_idx[g_idx_sort_indices]
405 -
406 - replace_tensor("g_idx", sorted_g_idx)
407 - replace_tensor("g_idx_sort_indices", g_idx_sort_indices)
408 -
409 - else:
410 - # Reset g_idx related tensors
411 - layer.g_idx = Parameter(
412 - torch.empty(0, dtype=torch.int, device=cur_device),
413 - requires_grad=False,
414 - )
415 - layer.g_idx_sort_indices = Parameter(
416 - torch.empty(0, dtype=torch.int, device=cur_device),
417 - requires_grad=False,
418 - )
419 -
420 - # Repack weights
421 - marlin_qweight = ops.gptq_marlin_repack(
422 - layer.qweight,
423 - layer.g_idx_sort_indices,
424 - part_size_k,
425 - part_size_n,
426 - self.quant_config.weight_bits,
427 - )
428 - replace_tensor("qweight", marlin_qweight)
429 -
430 - # Permute scales
431 - scales_size_k = part_size_k
432 - scales_size_n = part_size_n
433 - if self.quant_config.desc_act:
434 - scales_size_k = full_size_k
435 -
436 - marlin_scales = marlin_permute_scales(
437 - layer.scales,
438 - scales_size_k,
439 - scales_size_n,
440 - self.quant_config.group_size,
441 - self.quant_config.weight_bits,
442 - )
443 - replace_tensor("scales", marlin_scales)
444 -
445 - output = ops.gptq_marlin_gemm(
446 - reshaped_x,
447 - layer.qweight,
448 - layer.scales,
449 - layer.g_idx,
450 - layer.g_idx_sort_indices,
451 - layer.workspace,
452 - self.quant_config.weight_bits,
453 - size_m,
454 - part_size_n,
455 - part_size_k,
456 - layer.is_k_full,
457 - )
316 + out_shape = x.shape[:-1] + (layer.output_size_per_partition, )
317 +
318 + output = ops.gptq_marlin_gemm(reshaped_x,
319 + layer.qweight,
320 + layer.scales,
321 + g_idx=layer.g_idx,
322 + perm=layer.g_idx_sort_indices,
323 + workspace=layer.workspace,
324 + num_bits=self.quant_config.weight_bits,
325 + size_m=reshaped_x.shape[0],
326 + size_n=layer.output_size_per_partition,
327 + size_k=layer.input_size_per_partition,
328 + is_k_full=layer.is_k_full)
329
330 if bias is not None:
331 output.add_(bias) # In-place add
vllm/model_executor/layers/quantization/utils/marlin_24_perms.py deleted
-60
@@ -1,60 +0,0 @@
1 -"""This file is used for /tests and /benchmarks"""
2 -from typing import Dict, List
3 -
4 -import numpy
5 -import torch
6 -
7 -
8 -# Precompute permutations for Marlin24 weight and scale shuffling # noqa: E501
9 -#
10 -# Marlin works on [16*2,64] tiles. The goal of the permutations is to reorder the weight data so that it is compatible noqa: # noqa: E501
11 -# with the tensor-core format that is described here:
12 -# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
13 -#
14 -# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
15 -# (without the need to use ldmatrix instructions) # noqa: E501
16 -def get_perms_24(num_bits: int):
17 - perm_list: List[int] = []
18 - for i in range(32):
19 - perm1: List[int] = []
20 - col = i // 4
21 - col_o = col // 2
22 - for block in [0, 1]:
23 - for row in [
24 - 2 * (i % 4),
25 - 2 * (i % 4) + 1,
26 - 2 * (i % 4 + 4),
27 - 2 * (i % 4 + 4) + 1,
28 - ]:
29 - perm1.append(16 * row + col_o * 256 + 8 * (col % 2) +
30 - 4 * block)
31 - for j in range(4):
32 - perm_list.extend([p + 1 * j for p in perm1])
33 - perm = numpy.array(perm_list)
34 -
35 - if num_bits == 4:
36 - interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
37 - elif num_bits == 8:
38 - interleave = numpy.array([0, 2, 1, 3])
39 - else:
40 - raise ValueError("num_bits must be 4 or 8, got {}".format(num_bits))
41 -
42 - perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
43 - perm = torch.from_numpy(perm)
44 - scale_perm: List[int] = []
45 - for i in range(8):
46 - scale_perm.extend([i * 8 + j for j in [0, 4, 1, 5, 2, 6, 3, 7]])
47 - scale_perm_single: List[int] = []
48 - for i in range(8):
49 - scale_perm_single.extend([8 * i + j for j in [0, 1, 2, 3, 4, 5, 6, 7]])
50 - return perm, scale_perm, scale_perm_single
51 -
52 -
53 -marlin_24_perm: Dict[int, torch.Tensor] = {}
54 -marlin_24_scale_perm: Dict[int, List[int]] = {}
55 -marlin_24_scale_perm_single: Dict[int, List[int]] = {}
56 -for num_bits in [4, 8]:
57 - perm_24, scale_perm_24, scale_perm_single_24 = get_perms_24(num_bits)
58 - marlin_24_perm[num_bits] = perm_24
59 - marlin_24_scale_perm[num_bits] = scale_perm_24
60 - marlin_24_scale_perm_single[num_bits] = scale_perm_single_24
vllm/model_executor/layers/quantization/utils/marlin_perms.py deleted
-60
@@ -1,60 +0,0 @@
1 -"""This file is used for /tests and /benchmarks"""
2 -from typing import Dict, List
3 -
4 -import numpy
5 -import torch
6 -
7 -
8 -# Precompute permutations for Marlin weight and scale shuffling # noqa: E501
9 -#
10 -# Marlin works on [16,64] tiles. The goal of the permutations is to reorder the weight data so that it is compatible noqa: # noqa: E501
11 -# with the tensor-core format that is described here:
12 -# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
13 -#
14 -# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
15 -# (without the need to use ldmatrix instructions) # noqa: E501
16 -def get_perms(num_bits: int):
17 - perm_list: List[int] = []
18 - for i in range(32):
19 - perm1: List[int] = []
20 - col = i // 4
21 - for block in [0, 1]:
22 - for row in [
23 - 2 * (i % 4),
24 - 2 * (i % 4) + 1,
25 - 2 * (i % 4 + 4),
26 - 2 * (i % 4 + 4) + 1,
27 - ]:
28 - perm1.append(16 * row + col + 8 * block)
29 - for j in range(4):
30 - perm_list.extend([p + 256 * j for p in perm1])
31 -
32 - perm = numpy.array(perm_list)
33 -
34 - if num_bits == 4:
35 - interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
36 - elif num_bits == 8:
37 - interleave = numpy.array([0, 2, 1, 3])
38 - else:
39 - raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
40 -
41 - perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
42 - perm = torch.from_numpy(perm)
43 - scale_perm: List[int] = []
44 - for i in range(8):
45 - scale_perm.extend([i + 8 * j for j in range(8)])
46 - scale_perm_single: List[int] = []
47 - for i in range(4):
48 - scale_perm_single.extend(
49 - [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
50 - return perm, scale_perm, scale_perm_single
51 -
52 -
53 -marlin_perm: Dict[int, torch.Tensor] = {}
54 -marlin_scale_perm: Dict[int, List[int]] = {}
55 -marlin_scale_perm_single: Dict[int, List[int]] = {}
56 -for num_bits in [4, 8]:
57 - perm, scale_perm, scale_perm_single = get_perms(num_bits)
58 - marlin_perm[num_bits] = perm
59 - marlin_scale_perm[num_bits] = scale_perm
60 - marlin_scale_perm_single[num_bits] = scale_perm_single
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+133 -306
@@ -1,21 +1,9 @@
1 -"""This file is used for /tests and /benchmarks"""
2 -import random
3 -from typing import Optional
1 +from typing import List, Optional, Tuple
2
5 -import numpy
3 import torch
4
5 from vllm import _custom_ops as ops
9 -from vllm.model_executor.layers.quantization.utils.format_24 import (
10 - mask_creator, sparse_semi_structured_from_dense_cutlass)
11 -from vllm.model_executor.layers.quantization.utils.marlin_24_perms import (
12 - marlin_24_perm, marlin_24_scale_perm, marlin_24_scale_perm_single)
13 -from vllm.model_executor.layers.quantization.utils.marlin_perms import (
14 - marlin_perm, marlin_scale_perm, marlin_scale_perm_single)
15 -from vllm.model_executor.layers.quantization.utils.quant_utils import (
16 - get_pack_factor, quantize_weights, sort_weights)
6 from vllm.platforms import current_platform
18 -from vllm.utils import print_warning_once
7
8 GPTQ_MARLIN_TILE = 16
9 GPTQ_MARLIN_MIN_THREAD_N = 64
@@ -25,135 +13,110 @@ GPTQ_MARLIN_MAX_PARALLEL = 16
13 GPTQ_MARLIN_SUPPORTED_NUM_BITS = [4, 8]
14 GPTQ_MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
15 GPTQ_MARLIN_SUPPORTED_SYM = [True]
28 -
29 -
30 -def is_marlin_supported():
31 - capability = current_platform.get_device_capability()
32 - return capability[0] >= 8
33 -
34 -
35 -def apply_fp8_marlin_linear(
36 - input: torch.Tensor,
37 - weight: torch.Tensor,
38 - weight_scale: torch.Tensor,
39 - workspace: torch.Tensor,
40 - size_n: int,
41 - size_k: int,
42 - bias: Optional[torch.Tensor],
43 -) -> torch.Tensor:
44 - # For GPUs that lack FP8 hardware support, we can leverage the
45 - # Marlin kernel for fast weight-only FP8 quantization
46 -
47 - reshaped_x = input.reshape(-1, input.shape[-1])
48 - out_shape = input.shape[:-1] + (size_n, )
49 -
50 - output = ops.fp8_marlin_gemm(
51 - a=reshaped_x,
52 - b_q_weight=weight,
53 - b_scales=weight_scale,
54 - workspace=workspace,
55 - num_bits=8,
56 - size_m=reshaped_x.shape[0],
57 - size_n=size_n,
58 - size_k=size_k,
59 - )
60 -
61 - if bias is not None:
62 - output.add_(bias) # In-place add
63 -
64 - return output.reshape(out_shape)
65 -
66 -
67 -def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
68 - print_warning_once(
69 - "Your GPU does not have native support for FP8 computation but "
70 - "FP8 quantization is being used. Weight-only FP8 compression will "
71 - "be used leveraging the Marlin kernel. This may degrade "
72 - "performance for compute-heavy workloads.")
73 -
74 - part_size_n = layer.output_size_per_partition
75 - part_size_k = layer.input_size_per_partition
76 -
77 - device = layer.weight.device
78 -
79 - # WEIGHTS
80 - # Repack weights to gptq format (packed int32 elements)
81 - packed_gptq_qweight = pack_fp8_to_int32(layer.weight)
82 -
83 - # Repack weights to marlin format
84 - marlin_qweight = ops.gptq_marlin_repack(
85 - b_q_weight=packed_gptq_qweight,
86 - perm=torch.empty(0, dtype=torch.int, device=device),
87 - size_k=part_size_k,
88 - size_n=part_size_n,
89 - num_bits=8,
90 - )
91 - layer.weight = torch.nn.Parameter(marlin_qweight, requires_grad=False)
92 -
93 - # WEIGHT SCALES
94 - # Currently Marlin doesn't support per-tensor scales, so we
95 - # expand it to channelwise
96 - scales = layer.weight_scale.repeat(1, part_size_n).to(
97 - layer.orig_dtype).to(device)
98 - # Permute scales
99 - num_bits = 8
100 - marlin_scales = marlin_permute_scales(
101 - s=scales,
102 - size_k=part_size_k,
103 - size_n=part_size_n,
104 - group_size=-1,
105 - scale_perm=marlin_scale_perm[num_bits],
106 - scale_perm_single=marlin_scale_perm_single[num_bits])
107 - layer.weight_scale = torch.nn.Parameter(marlin_scales, requires_grad=False)
108 -
109 - # Allocate marlin workspace
110 - max_workspace_size = (part_size_n //
16 +GTPQ_MARLIN_UNSUPPORTED_GROUP_SIZE_ACT_ORDER = [-1]
17 +
18 +
19 +def check_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
20 + min_capability: int) -> bool:
21 +
22 + # If the capability of the device is too low, cannot convert.
23 + major, minor = current_platform.get_device_capability()
24 + device_capability = major * 10 + minor
25 + if device_capability < min_capability:
26 + return False
27 +
28 + return (device_capability >= min_capability
29 + and num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS
30 + and group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES
31 + and is_sym in GPTQ_MARLIN_SUPPORTED_SYM)
32 +
33 +
34 +def verify_marlin_supported(num_bits: int, group_size: Optional[int],
35 + is_sym: bool) -> None:
36 +
37 + if num_bits not in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
38 + raise ValueError(
39 + f"Marlin does not support weight_bits = {num_bits}. "
40 + f"Only weight_bits = {GPTQ_MARLIN_SUPPORTED_NUM_BITS} "
41 + "are supported.")
42 + if (group_size is None
43 + or group_size not in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES):
44 + raise ValueError(
45 + f"Marlin does not support group_size = {group_size}. "
46 + f"Only group_sizes = {GPTQ_MARLIN_SUPPORTED_GROUP_SIZES} "
47 + "are supported.")
48 + if is_sym not in GPTQ_MARLIN_SUPPORTED_SYM:
49 + raise ValueError(
50 + f"Marlin does not support is_sym = is_sym. "
51 + f"Only sym = {GPTQ_MARLIN_SUPPORTED_SYM} are supported.")
52 +
53 +
54 +def verify_marlin_supports_shape(output_size_per_partition: int,
55 + input_size_per_partition: int,
56 + input_size: int, group_size: int) -> None:
57 +
58 + # Validate output_size_per_partition
59 + if output_size_per_partition % GPTQ_MARLIN_MIN_THREAD_N != 0:
60 + raise ValueError(f"Weight output_size_per_partition = "
61 + f"{output_size_per_partition} is not divisible by "
62 + f" min_thread_n = {GPTQ_MARLIN_MIN_THREAD_N}. "
63 + "Consider reducing tensor_parallel_size or running "
64 + "with --quantization gptq.")
65 +
66 + # Validate input_size_per_partition
67 + if input_size_per_partition % GPTQ_MARLIN_MIN_THREAD_K != 0:
68 + raise ValueError(f"Weight input_size_per_partition = "
69 + f"{input_size_per_partition} is not divisible "
70 + f"by min_thread_k = {GPTQ_MARLIN_MIN_THREAD_K}. "
71 + "Consider reducing tensor_parallel_size or running "
72 + "with --quantization gptq.")
73 +
74 + if (group_size < input_size
75 + and input_size_per_partition % group_size != 0):
76 + raise ValueError(
77 + f"Weight input_size_per_partition = {input_size_per_partition}"
78 + f" is not divisible by group_size = {group_size}."
79 + "Consider reducing tensor_parallel_size or running "
80 + "with --quantization gptq.")
81 +
82 +
83 +def marlin_make_workspace(output_size_per_partition: int,
84 + device: torch.device) -> torch.Tensor:
85 + max_workspace_size = (output_size_per_partition //
86 GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL
112 - workspace = torch.zeros(max_workspace_size,
113 - dtype=torch.int,
114 - device=device,
115 - requires_grad=False)
116 -
117 - layer.workspace = workspace
118 -
119 -
120 -def marlin_permute_weights(q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE):
121 - assert q_w.shape == (size_k, size_n)
122 - assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
123 - assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
124 -
125 - # Permute weights to 16x64 marlin tiles
126 - q_w = q_w.reshape((size_k // tile, tile, size_n // tile, tile))
127 - q_w = q_w.permute((0, 2, 1, 3))
128 - q_w = q_w.reshape((size_k // tile, size_n * tile))
87
130 - q_w = q_w.reshape((-1, perm.numel()))[:, perm].reshape(q_w.shape)
88 + return torch.zeros(max_workspace_size,
89 + dtype=torch.int,
90 + device=device,
91 + requires_grad=False)
92
132 - return q_w
93
94 +def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:
95 + return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
96 + requires_grad=False)
97
135 -def marlin_weights(q_w, size_k, size_n, num_bits, perm):
136 - # Permute
137 - q_w = marlin_permute_weights(q_w, size_k, size_n, perm)
98
139 - # Pack
140 - pack_factor = get_pack_factor(num_bits)
141 - orig_device = q_w.device
99 +def marlin_sort_g_idx(
100 + g_idx: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
101 + g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)
102 + return g_idx[g_idx_sort_indices], g_idx_sort_indices
103
143 - q_w = q_w.cpu().numpy().astype(numpy.uint32)
104
145 - q_packed = numpy.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
146 - dtype=numpy.uint32)
147 - for i in range(pack_factor):
148 - q_packed |= q_w[:, i::pack_factor] << num_bits * i
105 +def get_scale_perms():
106 + scale_perm: List[int] = []
107 + for i in range(8):
108 + scale_perm.extend([i + 8 * j for j in range(8)])
109 + scale_perm_single: List[int] = []
110 + for i in range(4):
111 + scale_perm_single.extend(
112 + [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
113 + return scale_perm, scale_perm_single
114
150 - q_packed = torch.from_numpy(q_packed.astype(numpy.int32)).to(orig_device)
115
152 - return q_packed
116 +def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
117 + group_size: int) -> torch.Tensor:
118
154 -
155 -def marlin_permute_scales(s, size_k, size_n, group_size, scale_perm,
156 - scale_perm_single):
119 + scale_perm, scale_perm_single = get_scale_perms()
120 if group_size < size_k and group_size != -1:
121 s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
122 else:
@@ -163,180 +126,44 @@ def marlin_permute_scales(s, size_k, size_n, group_size, scale_perm,
126 return s
127
128
166 -def marlin_quantize(
167 - w: torch.Tensor,
168 - num_bits: int,
169 - group_size: int,
170 - act_order: bool,
171 -):
172 - size_k, size_n = w.shape
173 -
174 - # Normalize group_size
175 - if group_size == -1:
176 - group_size = size_k
177 - assert group_size <= size_k
178 -
179 - # Quantize (and apply act_order if provided)
180 - w_ref, q_w, s, g_idx, rand_perm = quantize_weights(w, num_bits, group_size,
181 - act_order)
182 -
183 - # For act_order, sort the "weights" and "g_idx" so that group ids are
184 - # increasing
185 - sort_indices = torch.empty(0, dtype=torch.int, device=w.device)
186 - if act_order:
187 - q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
188 -
189 - # Reformat to marlin
190 - marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits,
191 - marlin_perm[num_bits])
192 - marlin_s = marlin_permute_scales(s, size_k, size_n, group_size,
193 - marlin_scale_perm[num_bits],
194 - marlin_scale_perm_single[num_bits])
195 -
196 - # Create result
197 - res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm]
198 - for i in range(len(res_list)):
199 - res_list[i] = res_list[i].to(w.device)
200 -
201 - return res_list
202 -
203 -
204 -def inject_24(w, size_k, size_n):
205 - assert w.shape == (size_k, size_n)
206 -
207 - mask = mask_creator(w.t()).t().cuda().bool()
208 -
209 - return (mask * w).contiguous(), mask.contiguous()
210 -
211 -
212 -def check_24(w, num_rows_to_sample=50, _verbose=False):
213 - BLOCK_SIZE = 4
214 - MAX_NON_ZEROS = 2
215 -
216 - w = w.t().contiguous()
217 -
218 - print("check_24: w.shape = {}".format(w.shape))
219 -
220 - num_rows, num_cols = w.shape
221 - sampled_row_idxs = random.choices(range(num_rows), k=num_rows_to_sample)
222 - if _verbose:
223 - print(f"Sampled row idxs = {sampled_row_idxs}")
224 -
225 - total_segments = 0
226 - non_24_segments = 0
227 - for i in sampled_row_idxs:
228 - for j in range(0, num_cols - BLOCK_SIZE, BLOCK_SIZE):
229 - total_segments += 1
230 - block = w[i, j:j + BLOCK_SIZE]
231 - num_nonzero = torch.count_nonzero(block)
232 - if num_nonzero > MAX_NON_ZEROS:
233 - print("i = {} j = {} block = {}".format(i, j, block))
234 - non_24_segments += 1
235 -
236 - print(f"{non_24_segments} / {total_segments} do not have 2:4 structure.")
237 -
238 -
239 -def compress_quantized_24_weight(q_24, size_k, size_n, num_bits):
240 - assert q_24.shape == (size_k, size_n)
241 -
242 - # Remove zp to normalize over 0
243 - max_q_val = (1 << num_bits) - 1
244 - zp = (max_q_val + 1) // 2
245 - q_24_no_zp = q_24 - zp
246 -
247 - # Compress
248 - q_24_no_zp = q_24_no_zp.t().contiguous()
249 - q_24_no_zp_comp, meta = sparse_semi_structured_from_dense_cutlass(
250 - q_24_no_zp)
251 - q_24_no_zp_comp = q_24_no_zp_comp.t().contiguous()
252 -
253 - # Restore zp
254 - q_24_comp = q_24_no_zp_comp + zp
255 -
256 - # Resize meta to its actual shape (without moving any data)
257 - meta = meta.resize_(meta.shape[1] // 2, meta.shape[0] * 2)
258 -
259 - return q_24_comp, meta
260 -
261 -
262 -def marlin_24_quantize(
263 - w: torch.Tensor,
264 - num_bits: int,
265 - group_size: int,
266 -):
267 - size_k, size_n = w.shape
268 -
269 - # Normalize group_size
270 - if group_size == -1:
271 - group_size = size_k
272 - assert group_size <= size_k
273 -
274 - # Inject 2:4 sparsity
275 - w_24, mask_24 = inject_24(w, size_k, size_n)
276 -
277 - # Quantize
278 - w_24_ref, q_w_24, s, g_idx, rand_perm = quantize_weights(w_24,
279 - num_bits,
280 - group_size,
281 - act_order=False)
282 -
283 - # Compress quantized weight
284 - q_w_24_comp, meta = compress_quantized_24_weight(q_w_24, size_k, size_n,
285 - num_bits)
286 - size_k_comp = size_k // 2
287 -
288 - # Reformat to marlin
289 - marlin_24_q_w_comp = marlin_weights(q_w_24_comp, size_k_comp, size_n,
290 - num_bits, marlin_24_perm[num_bits])
291 - marlin_24_s = marlin_permute_scales(s, size_k, size_n, group_size,
292 - marlin_24_scale_perm[num_bits],
293 - marlin_24_scale_perm_single[num_bits])
294 -
295 - # Create result
296 - res_list = [w_24_ref, marlin_24_q_w_comp, meta, marlin_24_s]
297 - for i in range(len(res_list)):
298 - res_list[i] = res_list[i].to(w.device)
299 -
300 - return res_list
301 -
302 -
303 -def compute_max_diff(output, output_ref):
304 - return torch.mean(torch.abs(output - output_ref)) / torch.mean(
305 - torch.abs(output_ref))
306 -
307 -
308 -class MarlinWorkspace:
309 -
310 - def __init__(self, out_features, min_thread_n, max_parallel):
311 - assert (out_features % min_thread_n == 0), (
312 - "out_features = {} is undivisible by min_thread_n = {}".format(
313 - out_features, min_thread_n))
314 -
315 - max_workspace_size = ((out_features // min_thread_n) * max_parallel)
316 -
317 - self.scratch = torch.zeros(max_workspace_size,
318 - dtype=torch.int,
319 - device="cuda")
320 -
321 -
322 -def pack_fp8_to_int32(fp8_tensor: torch.Tensor) -> torch.Tensor:
323 - """
324 - Repack FP8 weights to gptq format (packed int32 elements)
325 - """
326 - assert fp8_tensor.dtype == torch.float8_e4m3fn
327 - assert fp8_tensor.shape[0] % 4 == 0
328 -
329 - # Reshape to prepare for packing
330 - reshaped = fp8_tensor.reshape(-1, 4, *fp8_tensor.shape[1:])
331 -
332 - # Convert fp8 to uint8 (byte) representation
333 - byte_tensor = reshaped.view(torch.uint8)
129 +# Newly generated tensors need to replace existing tensors that are
130 +# already registered as parameters by vLLM (and won't be freed)
131 +def replace_tensor(layer: torch.nn.Module, name: str,
132 + new_t: torch.Tensor) -> None:
133 + # It is important to use resize_() here since it ensures
134 + # the same buffer is reused
135 + getattr(layer, name).resize_(new_t.shape)
136 + getattr(layer, name).copy_(new_t)
137 + del new_t
138 +
139 +
140 +def apply_marlin_linear(input: torch.Tensor,
141 + weight: torch.Tensor,
142 + weight_scale: torch.Tensor,
143 + g_idx: torch.Tensor,
144 + g_idx_sort_indices: torch.Tensor,
145 + workspace: torch.Tensor,
146 + num_bits: int,
147 + output_size_per_partition: int,
148 + input_size_per_partition: int,
149 + is_k_full: bool,
150 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
151 + reshaped_x = input.reshape(-1, input.shape[-1])
152 + out_shape = input.shape[:-1] + (output_size_per_partition, )
153 +
154 + output = ops.gptq_marlin_gemm(reshaped_x,
155 + weight,
156 + weight_scale,
157 + g_idx,
158 + g_idx_sort_indices,
159 + workspace,
160 + num_bits,
161 + size_m=reshaped_x.shape[0],
162 + size_n=output_size_per_partition,
163 + size_k=input_size_per_partition,
164 + is_k_full=is_k_full)
165
335 - # Pack 4 uint8 values into one int32
336 - packed = (byte_tensor[:, 0].to(torch.int32) |
337 - (byte_tensor[:, 1].to(torch.int32) << 8) |
338 - (byte_tensor[:, 2].to(torch.int32) << 16) |
339 - (byte_tensor[:, 3].to(torch.int32) << 24))
166 + if bias is not None:
167 + output.add_(bias) # In-place add
168
341 - return packed.view(fp8_tensor.shape[0] // 4,
342 - *fp8_tensor.shape[1:]).contiguous()
169 + return output.reshape(out_shape)
vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py new
+109
@@ -0,0 +1,109 @@
1 +from typing import Optional
2 +
3 +import torch
4 +
5 +import vllm._custom_ops as ops
6 +from vllm.platforms import current_platform
7 +from vllm.utils import print_warning_once
8 +
9 +from .marlin_utils import marlin_make_workspace, marlin_permute_scales
10 +
11 +
12 +def is_fp8_marlin_supported():
13 + capability = current_platform.get_device_capability()
14 + return capability[0] >= 8
15 +
16 +
17 +def apply_fp8_marlin_linear(
18 + input: torch.Tensor,
19 + weight: torch.Tensor,
20 + weight_scale: torch.Tensor,
21 + workspace: torch.Tensor,
22 + size_n: int,
23 + size_k: int,
24 + bias: Optional[torch.Tensor],
25 +) -> torch.Tensor:
26 + # For GPUs that lack FP8 hardware support, we can leverage the
27 + # Marlin kernel for fast weight-only FP8 quantization
28 +
29 + reshaped_x = input.reshape(-1, input.shape[-1])
30 + out_shape = input.shape[:-1] + (size_n, )
31 +
32 + output = ops.fp8_marlin_gemm(
33 + a=reshaped_x,
34 + b_q_weight=weight,
35 + b_scales=weight_scale,
36 + workspace=workspace,
37 + num_bits=8,
38 + size_m=reshaped_x.shape[0],
39 + size_n=size_n,
40 + size_k=size_k,
41 + )
42 +
43 + if bias is not None:
44 + output.add_(bias) # In-place add
45 +
46 + return output.reshape(out_shape)
47 +
48 +
49 +def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
50 + print_warning_once(
51 + "Your GPU does not have native support for FP8 computation but "
52 + "FP8 quantization is being used. Weight-only FP8 compression will "
53 + "be used leveraging the Marlin kernel. This may degrade "
54 + "performance for compute-heavy workloads.")
55 +
56 + part_size_n = layer.output_size_per_partition
57 + part_size_k = layer.input_size_per_partition
58 +
59 + device = layer.weight.device
60 +
61 + # WORKSPACE
62 + layer.workspace = marlin_make_workspace(part_size_n, device)
63 +
64 + # WEIGHT
65 + # Repack weights to marlin format
66 + marlin_qweight = ops.gptq_marlin_repack(b_q_weight=pack_fp8_to_int32(
67 + layer.weight),
68 + perm=torch.empty(0,
69 + dtype=torch.int,
70 + device=device),
71 + size_k=part_size_k,
72 + size_n=part_size_n,
73 + num_bits=8)
74 + layer.weight = torch.nn.Parameter(marlin_qweight, requires_grad=False)
75 +
76 + # WEIGHT SCALES
77 + # Currently Marlin doesn't support per-tensor scales, so we
78 + # expand it to channelwise
79 + scales = layer.weight_scale.repeat(1, part_size_n).to(
80 + layer.orig_dtype).to(device)
81 + # Permute scales
82 + marlin_scales = marlin_permute_scales(s=scales,
83 + size_k=part_size_k,
84 + size_n=part_size_n,
85 + group_size=-1)
86 + layer.weight_scale = torch.nn.Parameter(marlin_scales, requires_grad=False)
87 +
88 +
89 +def pack_fp8_to_int32(fp8_tensor: torch.Tensor) -> torch.Tensor:
90 + """
91 + Repack FP8 weights to gptq format (packed int32 elements)
92 + """
93 + assert fp8_tensor.dtype == torch.float8_e4m3fn
94 + assert fp8_tensor.shape[0] % 4 == 0
95 +
96 + # Reshape to prepare for packing
97 + reshaped = fp8_tensor.reshape(-1, 4, *fp8_tensor.shape[1:])
98 +
99 + # Convert fp8 to uint8 (byte) representation
100 + byte_tensor = reshaped.view(torch.uint8)
101 +
102 + # Pack 4 uint8 values into one int32
103 + packed = (byte_tensor[:, 0].to(torch.int32) |
104 + (byte_tensor[:, 1].to(torch.int32) << 8) |
105 + (byte_tensor[:, 2].to(torch.int32) << 16) |
106 + (byte_tensor[:, 3].to(torch.int32) << 24))
107 +
108 + return packed.view(fp8_tensor.shape[0] // 4,
109 + *fp8_tensor.shape[1:]).contiguous()
vllm/model_executor/layers/quantization/utils/marlin_utils_test.py new
+120
@@ -0,0 +1,120 @@
1 +"""Utility functions used for tests and benchmarks"""
2 +
3 +from typing import List
4 +
5 +import numpy
6 +import torch
7 +
8 +from .marlin_utils import GPTQ_MARLIN_TILE, marlin_permute_scales
9 +from .quant_utils import get_pack_factor, quantize_weights, sort_weights
10 +
11 +
12 +class MarlinWorkspace:
13 +
14 + def __init__(self, out_features, min_thread_n, max_parallel):
15 + assert (out_features % min_thread_n == 0), (
16 + "out_features = {} is undivisible by min_thread_n = {}".format(
17 + out_features, min_thread_n))
18 +
19 + max_workspace_size = ((out_features // min_thread_n) * max_parallel)
20 +
21 + self.scratch = torch.zeros(max_workspace_size,
22 + dtype=torch.int,
23 + device="cuda")
24 +
25 +
26 +def marlin_permute_weights(q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE):
27 + assert q_w.shape == (size_k, size_n)
28 + assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
29 + assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
30 +
31 + # Permute weights to 16x64 marlin tiles
32 + q_w = q_w.reshape((size_k // tile, tile, size_n // tile, tile))
33 + q_w = q_w.permute((0, 2, 1, 3))
34 + q_w = q_w.reshape((size_k // tile, size_n * tile))
35 +
36 + q_w = q_w.reshape((-1, perm.numel()))[:, perm].reshape(q_w.shape)
37 +
38 + return q_w
39 +
40 +
41 +def marlin_weights(q_w, size_k, size_n, num_bits, perm):
42 + # Permute
43 + q_w = marlin_permute_weights(q_w, size_k, size_n, perm)
44 +
45 + # Pack
46 + pack_factor = get_pack_factor(num_bits)
47 + orig_device = q_w.device
48 +
49 + q_w = q_w.cpu().numpy().astype(numpy.uint32)
50 +
51 + q_packed = numpy.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
52 + dtype=numpy.uint32)
53 + for i in range(pack_factor):
54 + q_packed |= q_w[:, i::pack_factor] << num_bits * i
55 +
56 + q_packed = torch.from_numpy(q_packed.astype(numpy.int32)).to(orig_device)
57 +
58 + return q_packed
59 +
60 +
61 +def get_weight_perm(num_bits: int):
62 + perm_list: List[int] = []
63 + for i in range(32):
64 + perm1: List[int] = []
65 + col = i // 4
66 + for block in [0, 1]:
67 + for row in [
68 + 2 * (i % 4),
69 + 2 * (i % 4) + 1,
70 + 2 * (i % 4 + 4),
71 + 2 * (i % 4 + 4) + 1,
72 + ]:
73 + perm1.append(16 * row + col + 8 * block)
74 + for j in range(4):
75 + perm_list.extend([p + 256 * j for p in perm1])
76 +
77 + perm = numpy.array(perm_list)
78 +
79 + if num_bits == 4:
80 + interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
81 + elif num_bits == 8:
82 + interleave = numpy.array([0, 2, 1, 3])
83 + else:
84 + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
85 +
86 + perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
87 + perm = torch.from_numpy(perm)
88 + return perm
89 +
90 +
91 +def marlin_quantize(w: torch.Tensor, num_bits: int, group_size: int,
92 + act_order: bool):
93 + size_k, size_n = w.shape
94 +
95 + # Normalize group_size
96 + if group_size == -1:
97 + group_size = size_k
98 + assert group_size <= size_k
99 +
100 + # Quantize (and apply act_order if provided)
101 + w_ref, q_w, s, g_idx, rand_perm = quantize_weights(w, num_bits, group_size,
102 + act_order)
103 +
104 + # For act_order, sort the "weights" and "g_idx" so that group ids are
105 + # increasing
106 + sort_indices = torch.empty(0, dtype=torch.int, device=w.device)
107 + if act_order:
108 + q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
109 +
110 + # Reformat to marlin
111 + weight_perm = get_weight_perm(num_bits)
112 + marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm)
113 + marlin_s = marlin_permute_scales(s, size_k, size_n, group_size)
114 +
115 + # Create result
116 + res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm]
117 + for i in range(len(res_list)):
118 + res_list[i] = res_list[i].to(w.device)
119 +
120 + return res_list
vllm/model_executor/layers/quantization/utils/marlin_utils_test_24.py renamed
+160 -3
@@ -1,9 +1,14 @@
1 -#
2 -# Modified by Roberto Lopez Castro (roberto.lopez.castro@udc.es).
3 -#
1 +"""Utility functions used for tests and benchmarks"""
2
3 +import random
4 +from typing import List
5 +
6 +import numpy
7 import torch
8
9 +from .marlin_utils_test import marlin_weights
10 +from .quant_utils import quantize_weights
11 +
12
13 # This is PyTorch implementation of main part of reorder_meta()
14 # function, from tools/util/include/cutlass/util/host_reorder.h file
@@ -306,3 +311,155 @@ def mask_creator(tensor):
311 mask = w_b.scatter_(dim=1, index=index, value=0).reshape(tensor.shape)
312
313 return mask
314 +
315 +
316 +def inject_24(w, size_k, size_n):
317 + assert w.shape == (size_k, size_n)
318 +
319 + mask = mask_creator(w.t()).t().cuda().bool()
320 +
321 + return (mask * w).contiguous(), mask.contiguous()
322 +
323 +
324 +def check_24(w, num_rows_to_sample=50, _verbose=False):
325 + BLOCK_SIZE = 4
326 + MAX_NON_ZEROS = 2
327 +
328 + w = w.t().contiguous()
329 +
330 + print("check_24: w.shape = {}".format(w.shape))
331 +
332 + num_rows, num_cols = w.shape
333 + sampled_row_idxs = random.choices(range(num_rows), k=num_rows_to_sample)
334 + if _verbose:
335 + print(f"Sampled row idxs = {sampled_row_idxs}")
336 +
337 + total_segments = 0
338 + non_24_segments = 0
339 + for i in sampled_row_idxs:
340 + for j in range(0, num_cols - BLOCK_SIZE, BLOCK_SIZE):
341 + total_segments += 1
342 + block = w[i, j:j + BLOCK_SIZE]
343 + num_nonzero = torch.count_nonzero(block)
344 + if num_nonzero > MAX_NON_ZEROS:
345 + print("i = {} j = {} block = {}".format(i, j, block))
346 + non_24_segments += 1
347 +
348 + print(f"{non_24_segments} / {total_segments} do not have 2:4 structure.")
349 +
350 +
351 +def compress_quantized_24_weight(q_24, size_k, size_n, num_bits):
352 + assert q_24.shape == (size_k, size_n)
353 +
354 + # Remove zp to normalize over 0
355 + max_q_val = (1 << num_bits) - 1
356 + zp = (max_q_val + 1) // 2
357 + q_24_no_zp = q_24 - zp
358 +
359 + # Compress
360 + q_24_no_zp = q_24_no_zp.t().contiguous()
361 + q_24_no_zp_comp, meta = sparse_semi_structured_from_dense_cutlass(
362 + q_24_no_zp)
363 + q_24_no_zp_comp = q_24_no_zp_comp.t().contiguous()
364 +
365 + # Restore zp
366 + q_24_comp = q_24_no_zp_comp + zp
367 +
368 + # Resize meta to its actual shape (without moving any data)
369 + meta = meta.resize_(meta.shape[1] // 2, meta.shape[0] * 2)
370 +
371 + return q_24_comp, meta
372 +
373 +
374 +def get_scale_perms_24():
375 + scale_perm: List[int] = []
376 + for i in range(8):
377 + scale_perm.extend([i * 8 + j for j in [0, 4, 1, 5, 2, 6, 3, 7]])
378 + scale_perm_single: List[int] = []
379 + for i in range(8):
380 + scale_perm_single.extend([8 * i + j for j in [0, 1, 2, 3, 4, 5, 6, 7]])
381 + return scale_perm, scale_perm_single
382 +
383 +
384 +def get_weight_perm_24(num_bits: int):
385 + perm_list: List[int] = []
386 + for i in range(32):
387 + perm1: List[int] = []
388 + col = i // 4
389 + col_o = col // 2
390 + for block in [0, 1]:
391 + for row in [
392 + 2 * (i % 4),
393 + 2 * (i % 4) + 1,
394 + 2 * (i % 4 + 4),
395 + 2 * (i % 4 + 4) + 1,
396 + ]:
397 + perm1.append(16 * row + col_o * 256 + 8 * (col % 2) +
398 + 4 * block)
399 + for j in range(4):
400 + perm_list.extend([p + 1 * j for p in perm1])
401 + perm = numpy.array(perm_list)
402 +
403 + if num_bits == 4:
404 + interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
405 + elif num_bits == 8:
406 + interleave = numpy.array([0, 2, 1, 3])
407 + else:
408 + raise ValueError("num_bits must be 4 or 8, got {}".format(num_bits))
409 +
410 + perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
411 + perm = torch.from_numpy(perm)
412 + return perm
413 +
414 +
415 +def marlin_permute_scales_24(s: torch.Tensor, size_k: int, size_n: int,
416 + group_size: int) -> torch.Tensor:
417 +
418 + scale_perm, scale_perm_single = get_scale_perms_24()
419 + if group_size < size_k and group_size != -1:
420 + s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
421 + else:
422 + s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single]
423 + s = s.reshape((-1, size_n)).contiguous()
424 +
425 + return s
426 +
427 +
428 +def marlin_24_quantize(
429 + w: torch.Tensor,
430 + num_bits: int,
431 + group_size: int,
432 +):
433 + size_k, size_n = w.shape
434 +
435 + # Normalize group_size
436 + if group_size == -1:
437 + group_size = size_k
438 + assert group_size <= size_k
439 +
440 + # Inject 2:4 sparsity
441 + w_24, mask_24 = inject_24(w, size_k, size_n)
442 +
443 + # Quantize
444 + w_24_ref, q_w_24, s, g_idx, rand_perm = quantize_weights(w_24,
445 + num_bits,
446 + group_size,
447 + act_order=False)
448 +
449 + # Compress quantized weight
450 + q_w_24_comp, meta = compress_quantized_24_weight(q_w_24, size_k, size_n,
451 + num_bits)
452 + size_k_comp = size_k // 2
453 +
454 + # Reformat to marlin
455 + weight_perm = get_weight_perm_24(num_bits)
456 + marlin_24_q_w_comp = marlin_weights(q_w_24_comp, size_k_comp, size_n,
457 + num_bits, weight_perm)
458 + marlin_24_s = marlin_permute_scales_24(s, size_k, size_n, group_size)
459 +
460 + # Create result
461 + res_list = [w_24_ref, marlin_24_q_w_comp, meta, marlin_24_s]
462 + for i in range(len(res_list)):
463 + res_list[i] = res_list[i].to(w.device)
464 +
465 + return res_list