A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[ Misc ] More Cleanup of Marlin (#6359)
Co-authored-by: Robert Shaw <rshaw@neuralmagic.com>
Robert Shaw committed
Jul 13, 2024 at 06:21 UTC
babf52dade78ff3b1bea6cb6e9f4151dfd630251
3 files changed
+44
-48
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1
-1
@@ -3,7 +3,7 @@
3
# We use this for fp8, which HF does not support.
4
#
5
# Make sure you have lm-eval-harness installed:
6
-# pip install lm-eval==0.4.2
6
+# pip install lm-eval==0.4.3
7
8
usage() {
9
echo``
vllm/model_executor/layers/quantization/gptq_marlin.py
+31
-47
@@ -10,8 +10,9 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10
from vllm.model_executor.layers.quantization.base_config import (
11
QuantizationConfig)
12
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
13
- check_marlin_supported, marlin_make_empty_g_idx, marlin_make_workspace,
14
- marlin_permute_scales, marlin_sort_g_idx, replace_tensor,
13
+ apply_marlin_linear, check_marlin_supported, marlin_is_k_full,
14
+ marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
15
+ marlin_repeat_scales_on_all_ranks, marlin_sort_g_idx, replace_tensor,
16
verify_marlin_supported, verify_marlin_supports_shape)
17
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
18
@@ -145,6 +146,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
146
) -> None:
147
del output_size
148
output_size_per_partition = sum(output_partition_sizes)
149
+ is_row_parallel = input_size != input_size_per_partition
150
151
# Normalize group_size
152
if self.quant_config.group_size != -1:
@@ -158,32 +160,19 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
160
input_size=input_size,
161
group_size=group_size)
162
161
- # Detect sharding of scales/zp
162
-
163
- # By default, no sharding over "input dim"
164
- scales_and_zp_size = input_size // group_size
165
- scales_and_zp_input_dim = None
166
-
167
- if self.quant_config.desc_act:
168
- # Act-order case
169
- assert self.quant_config.group_size != -1
170
-
171
- is_k_full = input_size_per_partition == input_size
172
-
163
+ # Determine sharding
164
+ if marlin_repeat_scales_on_all_ranks(self.quant_config.desc_act,
165
+ self.quant_config.group_size,
166
+ is_row_parallel):
167
+ # By setting scale_dim == None, weight_loader will
168
+ # repeat the scales on each GPU in TP>1 case.
169
+ scales_and_zp_input_dim = None
170
+ scales_and_zp_size = input_size // group_size
171
else:
174
- # No act-order case
175
-
176
- # K is always full due to full alignment with
177
- # group-size and shard of scales/zp
178
- is_k_full = True
179
-
180
- # If this is a row-parallel case, then shard scales/zp
181
- if (input_size != input_size_per_partition
182
- and self.quant_config.group_size != -1):
183
- scales_and_zp_size = input_size_per_partition // group_size
184
- scales_and_zp_input_dim = 0
185
-
186
- # Init buffers
172
+ # By setting scale_dim == 0, weight_loader will
173
+ # shard the scales in TP>1 case.
174
+ scales_and_zp_input_dim = 0
175
+ scales_and_zp_size = input_size_per_partition // group_size
176
177
# Quantized weights
178
qweight = Parameter(
@@ -268,13 +257,15 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
257
layer.input_size_per_partition = input_size_per_partition
258
layer.output_size_per_partition = output_size_per_partition
259
layer.input_size = input_size
271
- layer.is_k_full = is_k_full
260
+ layer.is_k_full = marlin_is_k_full(self.quant_config.desc_act,
261
+ is_row_parallel)
262
263
# Checkpoints are serialized in AutoGPTQ format, which is different from the
264
# marlin format. This function is called after the weights are loaded.
265
# Here, we handle the repacking, including the activation reordering case.
266
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
267
device = layer.qweight.device
268
+
269
# Allocate marlin workspace
270
layer.workspace = marlin_make_workspace(
271
layer.output_size_per_partition, device)
@@ -312,22 +303,15 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
303
x: torch.Tensor,
304
bias: Optional[torch.Tensor] = None,
305
) -> torch.Tensor:
315
- reshaped_x = x.reshape(-1, x.shape[-1])
316
- out_shape = x.shape[:-1] + (layer.output_size_per_partition, )
317
-
318
- output = ops.gptq_marlin_gemm(reshaped_x,
319
- layer.qweight,
320
- layer.scales,
321
- g_idx=layer.g_idx,
322
- perm=layer.g_idx_sort_indices,
323
- workspace=layer.workspace,
324
- num_bits=self.quant_config.weight_bits,
325
- size_m=reshaped_x.shape[0],
326
- size_n=layer.output_size_per_partition,
327
- size_k=layer.input_size_per_partition,
328
- is_k_full=layer.is_k_full)
329
-
330
- if bias is not None:
331
- output.add_(bias) # In-place add
332
-
333
- return output.reshape(out_shape)
306
+ return apply_marlin_linear(
307
+ input=x,
308
+ weight=layer.qweight,
309
+ weight_scale=layer.scales,
310
+ g_idx=layer.g_idx,
311
+ g_idx_sort_indices=layer.g_idx_sort_indices,
312
+ workspace=layer.workspace,
313
+ num_bits=self.quant_config.weight_bits,
314
+ output_size_per_partition=layer.output_size_per_partition,
315
+ input_size_per_partition=layer.input_size_per_partition,
316
+ is_k_full=layer.is_k_full,
317
+ bias=bias)
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+12
@@ -91,6 +91,18 @@ def marlin_make_workspace(output_size_per_partition: int,
91
requires_grad=False)
92
93
94
+def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:
95
+ return (not act_order) or (act_order and not is_row_parallel)
96
+
97
+
98
+def marlin_repeat_scales_on_all_ranks(act_order: bool, group_size: int,
99
+ is_row_parallel: bool) -> bool:
100
+ # Need to repeat scales on every rank if act_ordering or
101
+ # channelwise and RowParallelLinear
102
+ is_channelwise = group_size == -1
103
+ return act_order or (is_channelwise and is_row_parallel)
104
+
105
+
106
def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:
107
return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
108
requires_grad=False)