@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / babf52dade
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ Misc ] More Cleanup of Marlin (#6359)

Co-authored-by: Robert Shaw <rshaw@neuralmagic.com>

Robert Shaw committed Jul 13, 2024 at 06:21 UTC babf52dade78ff3b1bea6cb6e9f4151dfd630251
3 files changed +44 -48
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1 -1
@@ -3,7 +3,7 @@
3 # We use this for fp8, which HF does not support.
4 #
5 # Make sure you have lm-eval-harness installed:
6 -# pip install lm-eval==0.4.2
6 +# pip install lm-eval==0.4.3
7
8 usage() {
9 echo``
vllm/model_executor/layers/quantization/gptq_marlin.py
+31 -47
@@ -10,8 +10,9 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
10 from vllm.model_executor.layers.quantization.base_config import (
11 QuantizationConfig)
12 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
13 - check_marlin_supported, marlin_make_empty_g_idx, marlin_make_workspace,
14 - marlin_permute_scales, marlin_sort_g_idx, replace_tensor,
13 + apply_marlin_linear, check_marlin_supported, marlin_is_k_full,
14 + marlin_make_empty_g_idx, marlin_make_workspace, marlin_permute_scales,
15 + marlin_repeat_scales_on_all_ranks, marlin_sort_g_idx, replace_tensor,
16 verify_marlin_supported, verify_marlin_supports_shape)
17 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
18
@@ -145,6 +146,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
146 ) -> None:
147 del output_size
148 output_size_per_partition = sum(output_partition_sizes)
149 + is_row_parallel = input_size != input_size_per_partition
150
151 # Normalize group_size
152 if self.quant_config.group_size != -1:
@@ -158,32 +160,19 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
160 input_size=input_size,
161 group_size=group_size)
162
161 - # Detect sharding of scales/zp
162 -
163 - # By default, no sharding over "input dim"
164 - scales_and_zp_size = input_size // group_size
165 - scales_and_zp_input_dim = None
166 -
167 - if self.quant_config.desc_act:
168 - # Act-order case
169 - assert self.quant_config.group_size != -1
170 -
171 - is_k_full = input_size_per_partition == input_size
172 -
163 + # Determine sharding
164 + if marlin_repeat_scales_on_all_ranks(self.quant_config.desc_act,
165 + self.quant_config.group_size,
166 + is_row_parallel):
167 + # By setting scale_dim == None, weight_loader will
168 + # repeat the scales on each GPU in TP>1 case.
169 + scales_and_zp_input_dim = None
170 + scales_and_zp_size = input_size // group_size
171 else:
174 - # No act-order case
175 -
176 - # K is always full due to full alignment with
177 - # group-size and shard of scales/zp
178 - is_k_full = True
179 -
180 - # If this is a row-parallel case, then shard scales/zp
181 - if (input_size != input_size_per_partition
182 - and self.quant_config.group_size != -1):
183 - scales_and_zp_size = input_size_per_partition // group_size
184 - scales_and_zp_input_dim = 0
185 -
186 - # Init buffers
172 + # By setting scale_dim == 0, weight_loader will
173 + # shard the scales in TP>1 case.
174 + scales_and_zp_input_dim = 0
175 + scales_and_zp_size = input_size_per_partition // group_size
176
177 # Quantized weights
178 qweight = Parameter(
@@ -268,13 +257,15 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
257 layer.input_size_per_partition = input_size_per_partition
258 layer.output_size_per_partition = output_size_per_partition
259 layer.input_size = input_size
271 - layer.is_k_full = is_k_full
260 + layer.is_k_full = marlin_is_k_full(self.quant_config.desc_act,
261 + is_row_parallel)
262
263 # Checkpoints are serialized in AutoGPTQ format, which is different from the
264 # marlin format. This function is called after the weights are loaded.
265 # Here, we handle the repacking, including the activation reordering case.
266 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
267 device = layer.qweight.device
268 +
269 # Allocate marlin workspace
270 layer.workspace = marlin_make_workspace(
271 layer.output_size_per_partition, device)
@@ -312,22 +303,15 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
303 x: torch.Tensor,
304 bias: Optional[torch.Tensor] = None,
305 ) -> torch.Tensor:
315 - reshaped_x = x.reshape(-1, x.shape[-1])
316 - out_shape = x.shape[:-1] + (layer.output_size_per_partition, )
317 -
318 - output = ops.gptq_marlin_gemm(reshaped_x,
319 - layer.qweight,
320 - layer.scales,
321 - g_idx=layer.g_idx,
322 - perm=layer.g_idx_sort_indices,
323 - workspace=layer.workspace,
324 - num_bits=self.quant_config.weight_bits,
325 - size_m=reshaped_x.shape[0],
326 - size_n=layer.output_size_per_partition,
327 - size_k=layer.input_size_per_partition,
328 - is_k_full=layer.is_k_full)
329 -
330 - if bias is not None:
331 - output.add_(bias) # In-place add
332 -
333 - return output.reshape(out_shape)
306 + return apply_marlin_linear(
307 + input=x,
308 + weight=layer.qweight,
309 + weight_scale=layer.scales,
310 + g_idx=layer.g_idx,
311 + g_idx_sort_indices=layer.g_idx_sort_indices,
312 + workspace=layer.workspace,
313 + num_bits=self.quant_config.weight_bits,
314 + output_size_per_partition=layer.output_size_per_partition,
315 + input_size_per_partition=layer.input_size_per_partition,
316 + is_k_full=layer.is_k_full,
317 + bias=bias)
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+12
@@ -91,6 +91,18 @@ def marlin_make_workspace(output_size_per_partition: int,
91 requires_grad=False)
92
93
94 +def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:
95 + return (not act_order) or (act_order and not is_row_parallel)
96 +
97 +
98 +def marlin_repeat_scales_on_all_ranks(act_order: bool, group_size: int,
99 + is_row_parallel: bool) -> bool:
100 + # Need to repeat scales on every rank if act_ordering or
101 + # channelwise and RowParallelLinear
102 + is_channelwise = group_size == -1
103 + return act_order or (is_channelwise and is_row_parallel)
104 +
105 +
106 def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:
107 return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
108 requires_grad=False)