@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e312e52b44
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Kernel] Add Exllama as a backend for compressed-tensors (#9395)

Lucas Wilkinson committed Oct 17, 2024 at 09:48 UTC e312e52b44f872896171f860a76805bfbd1d80bf
7 files changed +173 -16
vllm/envs.py
+9
@@ -66,6 +66,7 @@ if TYPE_CHECKING:
66 VLLM_SKIP_P2P_CHECK: bool = False
67 VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1: bool = False
68 VLLM_TORCH_COMPILE_LEVEL: int = 0
69 + VLLM_DISABLED_KERNELS: List[str] = []
70
71
72 def get_default_cache_root():
@@ -430,6 +431,14 @@ environment_variables: Dict[str, Callable[[], Any]] = {
431 "VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1":
432 lambda: os.environ.get("VLLM_ALLOW_DEPRECATED_BLOCK_MANAGER_V1", "0"
433 ) == "1",
434 +
435 + # List of quantization kernels that should be disabled, used for testing
436 + # and performance comparisons. Currently only affects MPLinearKernel
437 + # selection
438 + # (kernels: MacheteLinearKernel, MarlinLinearKernel, ExllamaLinearKernel)
439 + "VLLM_DISABLED_KERNELS":
440 + lambda: [] if "VLLM_DISABLED_KERNELS" not in os.environ else os.environ[
441 + "VLLM_DISABLED_KERNELS"].split(","),
442 }
443
444 # end-env-vars-definition
vllm/model_executor/layers/quantization/kernels/MPLinearKernel.py
+4
@@ -42,6 +42,10 @@ class MPLinearKernel(ABC):
42 self.config = c
43 self.w_q_name = w_q_param_name
44 self.w_s_name = w_s_param_name
45 + if c.zero_points:
46 + assert w_zp_param_name is not None
47 + if c.has_g_idx:
48 + assert w_gidx_param_name is not None
49 self.w_zp_name = w_zp_param_name
50 self.w_gidx_name = w_gidx_param_name
51
vllm/model_executor/layers/quantization/kernels/__init__.py
+5 -3
@@ -1,6 +1,8 @@
1 -import os
1 from typing import List, Optional, Type
2
3 +import vllm.envs as envs
4 +from vllm.model_executor.layers.quantization.kernels.exllama import (
5 + ExllamaLinearKernel)
6 from vllm.model_executor.layers.quantization.kernels.machete import (
7 MacheteLinearKernel)
8 from vllm.model_executor.layers.quantization.kernels.marlin import (
@@ -13,6 +15,7 @@ from vllm.platforms import current_platform
15 _POSSIBLE_KERNELS: List[Type[MPLinearKernel]] = [
16 MacheteLinearKernel,
17 MarlinLinearKernel,
18 + ExllamaLinearKernel,
19 ]
20
21
@@ -45,8 +48,7 @@ def choose_mp_linear_kernel(
48
49 failure_reasons = []
50 for kernel in _POSSIBLE_KERNELS:
48 - if kernel.__name__ in os.environ.get("VLLM_DISABLED_KERNELS", "")\
49 - .split(","):
51 + if kernel.__name__ in envs.VLLM_DISABLED_KERNELS:
52 failure_reasons.append(
53 f' {kernel.__name__} disabled by environment variable')
54 continue
vllm/model_executor/layers/quantization/kernels/exllama.py new
+140
@@ -0,0 +1,140 @@
1 +from typing import Optional, Tuple
2 +
3 +import torch
4 +
5 +from vllm import _custom_ops as ops
6 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
7 + pack_quantized_values_into_int32)
8 +from vllm.model_executor.parameter import (BasevLLMParameter,
9 + permute_param_layout_)
10 +from vllm.scalar_type import scalar_types
11 +
12 +from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
13 +
14 +
15 +class ExllamaLinearKernel(MPLinearKernel):
16 + SUPPORTED_QUANT_TYPES = [scalar_types.uint4b8, scalar_types.uint8b128]
17 + # In theory supports `scalar_types.uint2b2, scalar_types.uint3b4` too but
18 + # currently untested so not added to the list
19 +
20 + @classmethod
21 + def get_min_capability(cls) -> int:
22 + return 60
23 +
24 + @classmethod
25 + def can_implement(cls,
26 + c: MPLinearLayerConfig) -> Tuple[bool, Optional[str]]:
27 + if c.has_g_idx and\
28 + c.partition_weight_shape[0] != c.full_weight_shape[0]:
29 + return False, "Act reordering currently not supported by Exllama, "\
30 + "when the input features are partitioned across "\
31 + "devices"
32 +
33 + if c.partition_weight_shape[1] % (32 // c.weight_type.size_bits) != 0:
34 + return False, "Output features must be a multiple of the pack " \
35 + "factor (32 / num_bits) so that we can correctly " \
36 + "pack the zero points"
37 +
38 + if c.act_type != torch.float16:
39 + return False, "Exllama only supports float16 activations"
40 +
41 + if c.weight_type not in cls.SUPPORTED_QUANT_TYPES:
42 + return False, f"Quant type ({c.weight_type}) not supported by "\
43 + "Exllama, supported types are: "\
44 + f"{cls.SUPPORTED_QUANT_TYPES}"
45 +
46 + if c.full_weight_shape[0] % c.group_size != 0:
47 + return False, f"Group size ({c.group_size}) does not evenly divide"\
48 + " the number of input features "\
49 + f"({c.full_weight_shape[0]})"
50 +
51 + return True, None
52 +
53 + def process_weights_after_loading(self, layer: torch.nn.Module):
54 + c = self.config
55 +
56 + # For Exllama, we need to set a zero-point tensor if there is not one
57 + if not c.zero_points:
58 + self.w_zp_name = "qzeros"
59 + device = getattr(layer, self.w_q_name).device
60 + groups = c.partition_weight_shape[0] // c.group_size
61 + out_features = c.partition_weight_shape[1]
62 +
63 + if c.weight_type.has_bias():
64 + # if the type has a bias we have to create a zeros tensor that
65 + # contains the bias values repeated for each group (-1 due to
66 + # a bug in the original GPTQ checkpoint format leading to
67 + # exllama kernel adding 1 to the zero points during inference)
68 + # Documentation of the bug can be found here:
69 + # https://garden.danieldk.eu/GPTQ-Checkpoint-Format
70 + zeros = torch.full((groups, out_features),
71 + c.weight_type.bias - 1,
72 + dtype=torch.int32,
73 + device=device)
74 + else:
75 + raise NotImplementedError(
76 + "A 0 zero-point is not supported by Exllama due to "
77 + "a bug in the original GPTQ checkpoint format leading to "
78 + "exllama kernel adding 1 to the zero points during "
79 + "inference")
80 + zeros = pack_quantized_values_into_int32(zeros,
81 + c.weight_type,
82 + packed_dim=1)
83 + setattr(layer, self.w_zp_name,
84 + torch.nn.Parameter(zeros, requires_grad=False))
85 +
86 + if c.has_g_idx:
87 +
88 + def transform_w_g_idx(x):
89 + # Exllama wants the permutation array instead of the group
90 + # indices
91 + return torch.argsort(x).to(torch.int)
92 +
93 + self._transform_param(layer, self.w_gidx_name, transform_w_g_idx)
94 + else:
95 + self.w_gidx_name = "g_idx"
96 + empty_g_idx = torch.nn.Parameter(torch.empty((0, ),
97 + dtype=torch.int,
98 + device=device),
99 + requires_grad=False)
100 + setattr(layer, self.w_gidx_name, empty_g_idx)
101 +
102 + def transform_w_q(x):
103 + assert isinstance(x, BasevLLMParameter)
104 + assert self.w_gidx_name is not None
105 + g_idx = getattr(layer, self.w_gidx_name)
106 +
107 + permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
108 + x_cont = x.data.contiguous()
109 + ops.gptq_shuffle(x_cont, g_idx, c.weight_type.size_bits)
110 + return x_cont
111 +
112 + def transform_w_s(x):
113 + assert isinstance(x, BasevLLMParameter)
114 + permute_param_layout_(x, input_dim=0, output_dim=1)
115 + x.data = x.data.contiguous()
116 + return x.to(dtype=c.act_type)
117 +
118 + # Repack weights and scales for Machete
119 + self._transform_param(layer, self.w_q_name, transform_w_q)
120 + self._transform_param(layer, self.w_s_name, transform_w_s)
121 +
122 + def apply_weights(self,
123 + layer: torch.nn.Module,
124 + x: torch.Tensor,
125 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
126 + c = self.config
127 +
128 + x_2d = x.reshape(-1, x.shape[-1])
129 + out_shape = x.shape[:-1] + (c.partition_weight_shape[1], )
130 +
131 + w_q, w_s, w_zp, w_g_idx = self._get_weight_params(layer)
132 +
133 + assert w_zp is not None, "Zero points are required by Exllama"
134 + assert w_g_idx is not None, "Group index is required by Exllama"
135 + output = ops.gptq_gemm(x_2d, w_q, w_zp, w_s, w_g_idx, True,
136 + c.weight_type.size_bits)
137 +
138 + if bias is not None:
139 + output.add_(bias)
140 + return output.reshape(out_shape)
vllm/model_executor/layers/quantization/kernels/machete.py
+7 -7
@@ -8,7 +8,7 @@ from vllm.model_executor.layers.quantization.utils.machete_utils import (
8 MACHETE_SUPPORTED_GROUP_SIZES, check_machete_supports_shape,
9 query_machete_supported_quant_types)
10 from vllm.model_executor.layers.quantization.utils.quant_utils import (
11 - pack_weights_into_int32, unpack_weights_into_int32)
11 + pack_quantized_values_into_int32, unpack_quantized_values_into_int32)
12 from vllm.model_executor.parameter import (BasevLLMParameter,
13 permute_param_layout_)
14
@@ -71,13 +71,13 @@ class MacheteLinearKernel(MPLinearKernel):
71 assert isinstance(x, BasevLLMParameter)
72 permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
73 if c.has_g_idx:
74 - x_unpacked = unpack_weights_into_int32(x.data,
75 - c.weight_type,
76 - packed_dim=0)
74 + x_unpacked = unpack_quantized_values_into_int32(x.data,
75 + c.weight_type,
76 + packed_dim=0)
77 x_perm = x_unpacked[perm, :]
78 - x.data = pack_weights_into_int32(x_perm,
79 - c.weight_type,
80 - packed_dim=0)
78 + x.data = pack_quantized_values_into_int32(x_perm,
79 + c.weight_type,
80 + packed_dim=0)
81 x.data = ops.machete_prepack_B(x.data.t().contiguous().t(),
82 self.config.weight_type)
83 return x
vllm/model_executor/layers/quantization/utils/quant_utils.py
+6 -6
@@ -20,9 +20,9 @@ FUSED_LAYER_NAME_MAPPING = {
20 }
21
22
23 -def pack_weights_into_int32(w_q: torch.Tensor,
24 - wtype: ScalarType,
25 - packed_dim: int = 0):
23 +def pack_quantized_values_into_int32(w_q: torch.Tensor,
24 + wtype: ScalarType,
25 + packed_dim: int = 0):
26 # move dim to pack to the end
27 perm = (*[i for i in range(len(w_q.shape)) if i != packed_dim], packed_dim)
28 inv_perm = tuple(perm.index(i) for i in range(len(perm)))
@@ -42,9 +42,9 @@ def pack_weights_into_int32(w_q: torch.Tensor,
42 return res.permute(inv_perm)
43
44
45 -def unpack_weights_into_int32(w_q: torch.Tensor,
46 - wtype: ScalarType,
47 - packed_dim: int = 0):
45 +def unpack_quantized_values_into_int32(w_q: torch.Tensor,
46 + wtype: ScalarType,
47 + packed_dim: int = 0):
48 # move dim to pack to the end
49 perm = (*[i for i in range(len(w_q.shape)) if i != packed_dim], packed_dim)
50 inv_perm = tuple(perm.index(i) for i in range(len(perm)))
vllm/scalar_type.py
+2
@@ -27,6 +27,8 @@ class scalar_types:
27 float6_e3m2f = ScalarType.float_(3, 2, True, NanRepr.NONE.value)
28
29 # "gptq" types
30 + uint2b2 = ScalarType.uint(2, 2)
31 + uint3b4 = ScalarType.uint(3, 4)
32 uint4b8 = ScalarType.uint(4, 8)
33 uint8b128 = ScalarType.uint(8, 128)
34