@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b98c62ba49
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix] Fix GGUF inference with FP16 unquantized checkpoint (#10675)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 28, 2024 at 02:43 UTC b98c62ba4947b93673c522b13464854acf8090a4
1 file changed +60 -9
vllm/model_executor/layers/quantization/gguf.py
+60 -9
@@ -2,6 +2,7 @@ from typing import Any, Dict, List, Optional
2
3 import gguf
4 import torch
5 +from gguf import GGMLQuantizationType as WeightType
6 from torch.nn.parameter import Parameter, UninitializedParameter
7
8 from vllm import _custom_ops as ops
@@ -49,19 +50,65 @@ class GGUFConfig(QuantizationConfig):
50 return None
51
52
53 +UNQUANTIZED_TYPES = {WeightType.F32, WeightType.F16, WeightType.BF16}
54 +STANDARD_QUANT_TYPES = {
55 + WeightType.Q4_0,
56 + WeightType.Q4_1,
57 + WeightType.Q5_0,
58 + WeightType.Q5_1,
59 + WeightType.Q8_0,
60 + WeightType.Q8_1,
61 +}
62 +KQUANT_TYPES = {
63 + WeightType.Q2_K,
64 + WeightType.Q3_K,
65 + WeightType.Q4_K,
66 + WeightType.Q5_K,
67 + WeightType.Q6_K,
68 +}
69 +IMATRIX_QUANT_TYPES = {
70 + WeightType.IQ1_M,
71 + WeightType.IQ1_S,
72 + WeightType.IQ2_XXS,
73 + WeightType.IQ2_XS,
74 + WeightType.IQ2_S,
75 + WeightType.IQ3_XXS,
76 + WeightType.IQ3_S,
77 + WeightType.IQ4_XS,
78 + WeightType.IQ4_NL,
79 +}
80 +# TODO(Isotr0py): Currently, we don't have MMQ kernel for I-Matrix quantization.
81 +# Consolidate DEQUANT_TYPES, MMVQ_QUANT_TYPES and MMQ_QUANT_TYPES after we add
82 +# MMQ kernel for I-Matrix quantization.
83 +DEQUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
84 +MMVQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
85 +MMQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES
86 +
87 +
88 def _fuse_mul_mat(x: torch.Tensor, qweight: torch.Tensor,
89 qweight_type: int) -> torch.Tensor:
54 - # use dequantize mulmat for IQmatrix, mmq for k-quants
55 - if x.shape[0] == 1:
56 - # enable mmvq in contiguous batching
90 + # there is no need to call any kernel for fp16/bf16
91 + if qweight_type in UNQUANTIZED_TYPES:
92 + return x @ qweight.T
93 + # enable MMVQ in contiguous batching with batch_size=1
94 + if x.shape[0] == 1 and qweight_type in MMVQ_QUANT_TYPES:
95 y = ops.ggml_mul_mat_vec_a8(qweight, x, qweight_type, qweight.shape[0])
58 - elif qweight_type >= 16:
96 + # Use MMQ Kernel if it's available (standard + k-quants)
97 + elif qweight_type in MMQ_QUANT_TYPES:
98 + y = ops.ggml_mul_mat_a8(qweight, x, qweight_type, qweight.shape[0])
99 + # If there is no available MMQ kernel, fallback to dequantize
100 + elif qweight_type in DEQUANT_TYPES:
101 block_size, type_size = gguf.GGML_QUANT_SIZES[qweight_type]
102 shape = (qweight.shape[0], qweight.shape[1] // type_size * block_size)
103 weight = ops.ggml_dequantize(qweight, qweight_type, *shape)
104 y = x @ weight.T
105 else:
64 - y = ops.ggml_mul_mat_a8(qweight, x, qweight_type, qweight.shape[0])
106 + # Raise an error if the quantization type is not supported.
107 + # Might be useful if llama.cpp adds a new quantization type.
108 + # Wrap to GGMLQuantizationType IntEnum to make sure it's a valid type.
109 + qweight_type = WeightType(qweight_type)
110 + raise NotImplementedError(
111 + f"Unsupported GGUF quantization type: {qweight_type}")
112 return y
113
114
@@ -121,9 +168,9 @@ class GGUFLinearMethod(LinearMethodBase):
168 shard_id = ["q", "k", "v"] if "q" in shard_id else shard_id
169 qweight = layer.qweight.unbind(0)
170 result = []
124 - for id in shard_id:
125 - q_idx = layer.qweight.shard_id_map[id]
126 - qweight_type = layer.qweight_type.shard_weight_type[id]
171 + for idx in shard_id:
172 + q_idx = layer.qweight.shard_id_map[idx]
173 + qweight_type = layer.qweight_type.shard_weight_type[idx]
174 result.append(_fuse_mul_mat(x, qweight[q_idx], qweight_type))
175 out = torch.cat(result, axis=1)
176 else:
@@ -163,9 +210,13 @@ class GGUFUninitializedParameter(UninitializedParameter):
210 data_container: List[torch.Tensor]
211
212 def materialize_nested(self) -> Parameter:
213 + dtype = {data.dtype for data in self.data_container}
214 + assert len(dtype) == 1, ValueError(
215 + f"Data container has mixed dtypes: {dtype}")
216 + dtype = next(iter(dtype))
217 nested_data = torch.nested.nested_tensor(self.data_container,
218 device=self.device,
168 - dtype=torch.uint8)
219 + dtype=dtype)
220 self.data_container.clear()
221 param = torch.Tensor._make_subclass(self.cls_to_become,
222 nested_data,