| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | |
| 5 | import torch |
| 6 | |
| 7 | from vllm import _custom_ops as ops |
| 8 | from vllm.model_executor.layers.quantization.utils.quant_utils import ( |
| 9 | pack_quantized_values_into_int32, |
| 10 | ) |
| 11 | from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_ |
| 12 | from vllm.platforms import current_platform |
| 13 | from vllm.scalar_type import scalar_types |
| 14 | |
| 15 | from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig |
| 16 | |
| 17 | |
| 18 | class ExllamaLinearKernel(MPLinearKernel): |
| 19 | SUPPORTED_QUANT_TYPES = [scalar_types.uint4b8, scalar_types.uint8b128] |
| 20 | # In theory supports `scalar_types.uint2b2, scalar_types.uint3b4` too but |
| 21 | # currently untested so not added to the list |
| 22 | |
| 23 | @classmethod |
| 24 | def get_min_capability(cls) -> int: |
| 25 | return 60 |
| 26 | |
| 27 | @classmethod |
| 28 | def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]: |
| 29 | if not current_platform.is_cuda_alike(): |
| 30 | return ( |
| 31 | False, |
| 32 | "Exllama is only supported on CUDA and ROCm", |
| 33 | ) |
| 34 | |
| 35 | if c.partition_weight_shape[1] % (32 // c.weight_type.size_bits) != 0: |
| 36 | return ( |
| 37 | False, |
| 38 | "Output features must be a multiple of the pack " |
| 39 | "factor (32 / num_bits) so that we can correctly " |
| 40 | "pack the zero points", |
| 41 | ) |
| 42 | |
| 43 | if c.act_type != torch.float16: |
| 44 | return False, "Exllama only supports float16 activations" |
| 45 | |
| 46 | if c.weight_type not in cls.SUPPORTED_QUANT_TYPES: |
| 47 | return ( |
| 48 | False, |
| 49 | f"Quant type ({c.weight_type}) not supported by " |
| 50 | "Exllama, supported types are: " |
| 51 | f"{cls.SUPPORTED_QUANT_TYPES}", |
| 52 | ) |
| 53 | |
| 54 | if c.group_size <= 0: |
| 55 | return ( |
| 56 | False, |
| 57 | f"Group size ({c.group_size}) must be positive, " |
| 58 | "Exllama does not support channelwise quantization", |
| 59 | ) |
| 60 | |
| 61 | if c.full_weight_shape[0] % c.group_size != 0: |
| 62 | return ( |
| 63 | False, |
| 64 | f"Group size ({c.group_size}) does not evenly divide" |
| 65 | " the number of input features " |
| 66 | f"({c.full_weight_shape[0]})", |
| 67 | ) |
| 68 | |
| 69 | return True, None |
| 70 | |
| 71 | def process_weights_after_loading(self, layer: torch.nn.Module): |
| 72 | c = self.config |
| 73 | |
| 74 | # For Exllama, we need to set a zero-point tensor if there is not one |
| 75 | if not c.zero_points: |
| 76 | self.w_zp_name = "qzeros" |
| 77 | device = getattr(layer, self.w_q_name).device |
| 78 | groups = c.partition_weight_shape[0] // c.group_size |
| 79 | out_features = c.partition_weight_shape[1] |
| 80 | |
| 81 | if c.weight_type.has_bias(): |
| 82 | # if the type has a bias we have to create a zeros tensor that |
| 83 | # contains the bias values repeated for each group (-1 due to |
| 84 | # a bug in the original GPTQ checkpoint format leading to |
| 85 | # exllama kernel adding 1 to the zero points during inference) |
| 86 | # Documentation of the bug can be found here: |
| 87 | # https://garden.danieldk.eu/GPTQ-Checkpoint-Format |
| 88 | zeros = torch.full( |
| 89 | (groups, out_features), |
| 90 | c.weight_type.bias - 1, |
| 91 | dtype=torch.int32, |
| 92 | device=device, |
| 93 | ) |
| 94 | else: |
| 95 | raise NotImplementedError( |
| 96 | "A 0 zero-point is not supported by Exllama due to " |
| 97 | "a bug in the original GPTQ checkpoint format leading to " |
| 98 | "exllama kernel adding 1 to the zero points during " |
| 99 | "inference" |
| 100 | ) |
| 101 | zeros = pack_quantized_values_into_int32(zeros, c.weight_type, packed_dim=1) |
| 102 | setattr( |
| 103 | layer, self.w_zp_name, torch.nn.Parameter(zeros, requires_grad=False) |
| 104 | ) |
| 105 | |
| 106 | def transform_w_q(x): |
| 107 | assert isinstance(x, BasevLLMParameter) |
| 108 | permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0) |
| 109 | x_cont = x.data.contiguous() |
| 110 | ops.gptq_shuffle(x_cont, c.weight_type.size_bits) |
| 111 | return x_cont |
| 112 | |
| 113 | def transform_w_s(x): |
| 114 | assert isinstance(x, BasevLLMParameter) |
| 115 | permute_param_layout_(x, input_dim=0, output_dim=1) |
| 116 | x.data = x.data.contiguous() |
| 117 | return x.to(dtype=c.act_type) |
| 118 | |
| 119 | # Repack weights and scales for Machete |
| 120 | self._transform_param(layer, self.w_q_name, transform_w_q) |
| 121 | self._transform_param(layer, self.w_s_name, transform_w_s) |
| 122 | |
| 123 | def apply_weights( |
| 124 | self, |
| 125 | layer: torch.nn.Module, |
| 126 | x: torch.Tensor, |
| 127 | bias: torch.Tensor | None = None, |
| 128 | ) -> torch.Tensor: |
| 129 | c = self.config |
| 130 | |
| 131 | x_2d = x.reshape(-1, x.shape[-1]) |
| 132 | out_shape = x.shape[:-1] + (c.partition_weight_shape[1],) |
| 133 | |
| 134 | w_q, w_s, w_zp = self._get_weight_params(layer) |
| 135 | # gptq_gemm supports GPTQv2 format by passing use_v2_format=True. |
| 136 | # However, the MPLinearLayerConfig doesn't contain format info. |
| 137 | # So hardcode GPTQv1 format here, to keep its behavior unchanged. |
| 138 | use_v2_format = False |
| 139 | |
| 140 | assert w_zp is not None, "Zero points are required by Exllama" |
| 141 | output = ops.gptq_gemm( |
| 142 | x_2d, w_q, w_zp, w_s, True, use_v2_format, c.weight_type.size_bits |
| 143 | ) |
| 144 | |
| 145 | if bias is not None: |
| 146 | output.add_(bias) |
| 147 | return output.reshape(out_shape) |