A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix] Fix GGUF inference with FP16 unquantized checkpoint (#10675)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 28, 2024 at 02:43 UTC
b98c62ba4947b93673c522b13464854acf8090a4
1 file changed
+60
-9
vllm/model_executor/layers/quantization/gguf.py
+60
-9
@@ -2,6 +2,7 @@ from typing import Any, Dict, List, Optional
2
3
import gguf
4
import torch
5
+from gguf import GGMLQuantizationType as WeightType
6
from torch.nn.parameter import Parameter, UninitializedParameter
7
8
from vllm import _custom_ops as ops
@@ -49,19 +50,65 @@ class GGUFConfig(QuantizationConfig):
50
return None
51
52
53
+UNQUANTIZED_TYPES = {WeightType.F32, WeightType.F16, WeightType.BF16}
54
+STANDARD_QUANT_TYPES = {
55
+ WeightType.Q4_0,
56
+ WeightType.Q4_1,
57
+ WeightType.Q5_0,
58
+ WeightType.Q5_1,
59
+ WeightType.Q8_0,
60
+ WeightType.Q8_1,
61
+}
62
+KQUANT_TYPES = {
63
+ WeightType.Q2_K,
64
+ WeightType.Q3_K,
65
+ WeightType.Q4_K,
66
+ WeightType.Q5_K,
67
+ WeightType.Q6_K,
68
+}
69
+IMATRIX_QUANT_TYPES = {
70
+ WeightType.IQ1_M,
71
+ WeightType.IQ1_S,
72
+ WeightType.IQ2_XXS,
73
+ WeightType.IQ2_XS,
74
+ WeightType.IQ2_S,
75
+ WeightType.IQ3_XXS,
76
+ WeightType.IQ3_S,
77
+ WeightType.IQ4_XS,
78
+ WeightType.IQ4_NL,
79
+}
80
+# TODO(Isotr0py): Currently, we don't have MMQ kernel for I-Matrix quantization.
81
+# Consolidate DEQUANT_TYPES, MMVQ_QUANT_TYPES and MMQ_QUANT_TYPES after we add
82
+# MMQ kernel for I-Matrix quantization.
83
+DEQUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
84
+MMVQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
85
+MMQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES
86
+
87
+
88
def _fuse_mul_mat(x: torch.Tensor, qweight: torch.Tensor,
89
qweight_type: int) -> torch.Tensor:
54
- # use dequantize mulmat for IQmatrix, mmq for k-quants
55
- if x.shape[0] == 1:
56
- # enable mmvq in contiguous batching
90
+ # there is no need to call any kernel for fp16/bf16
91
+ if qweight_type in UNQUANTIZED_TYPES:
92
+ return x @ qweight.T
93
+ # enable MMVQ in contiguous batching with batch_size=1
94
+ if x.shape[0] == 1 and qweight_type in MMVQ_QUANT_TYPES:
95
y = ops.ggml_mul_mat_vec_a8(qweight, x, qweight_type, qweight.shape[0])
58
- elif qweight_type >= 16:
96
+ # Use MMQ Kernel if it's available (standard + k-quants)
97
+ elif qweight_type in MMQ_QUANT_TYPES:
98
+ y = ops.ggml_mul_mat_a8(qweight, x, qweight_type, qweight.shape[0])
99
+ # If there is no available MMQ kernel, fallback to dequantize
100
+ elif qweight_type in DEQUANT_TYPES:
101
block_size, type_size = gguf.GGML_QUANT_SIZES[qweight_type]
102
shape = (qweight.shape[0], qweight.shape[1] // type_size * block_size)
103
weight = ops.ggml_dequantize(qweight, qweight_type, *shape)
104
y = x @ weight.T
105
else:
64
- y = ops.ggml_mul_mat_a8(qweight, x, qweight_type, qweight.shape[0])
106
+ # Raise an error if the quantization type is not supported.
107
+ # Might be useful if llama.cpp adds a new quantization type.
108
+ # Wrap to GGMLQuantizationType IntEnum to make sure it's a valid type.
109
+ qweight_type = WeightType(qweight_type)
110
+ raise NotImplementedError(
111
+ f"Unsupported GGUF quantization type: {qweight_type}")
112
return y
113
114
@@ -121,9 +168,9 @@ class GGUFLinearMethod(LinearMethodBase):
168
shard_id = ["q", "k", "v"] if "q" in shard_id else shard_id
169
qweight = layer.qweight.unbind(0)
170
result = []
124
- for id in shard_id:
125
- q_idx = layer.qweight.shard_id_map[id]
126
- qweight_type = layer.qweight_type.shard_weight_type[id]
171
+ for idx in shard_id:
172
+ q_idx = layer.qweight.shard_id_map[idx]
173
+ qweight_type = layer.qweight_type.shard_weight_type[idx]
174
result.append(_fuse_mul_mat(x, qweight[q_idx], qweight_type))
175
out = torch.cat(result, axis=1)
176
else:
@@ -163,9 +210,13 @@ class GGUFUninitializedParameter(UninitializedParameter):
210
data_container: List[torch.Tensor]
211
212
def materialize_nested(self) -> Parameter:
213
+ dtype = {data.dtype for data in self.data_container}
214
+ assert len(dtype) == 1, ValueError(
215
+ f"Data container has mixed dtypes: {dtype}")
216
+ dtype = next(iter(dtype))
217
nested_data = torch.nested.nested_tensor(self.data_container,
218
device=self.device,
168
- dtype=torch.uint8)
219
+ dtype=dtype)
220
self.data_container.clear()
221
param = torch.Tensor._make_subclass(self.cls_to_become,
222
nested_data,