A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Quantization] Allow GGUF quantization to skip unquantized layer (#23188)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Aug 23, 2025 at 03:04 UTC
4645024d3ad4caf748dbbd0541d1a0835b95bd0a
3 files changed
+36
-3
vllm/model_executor/layers/quantization/gguf.py
+11
-2
@@ -13,7 +13,8 @@ from vllm.logger import init_logger
13
from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,
14
FusedMoEConfig,
15
FusedMoEMethodBase)
16
-from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
16
+from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
17
+ UnquantizedLinearMethod)
18
from vllm.model_executor.layers.quantization import QuantizationMethods
19
from vllm.model_executor.layers.quantization.base_config import (
20
QuantizationConfig, QuantizeMethodBase)
@@ -28,8 +29,10 @@ logger = init_logger(__name__)
29
class GGUFConfig(QuantizationConfig):
30
"""Config class for GGUF."""
31
31
- def __init__(self, ) -> None:
32
+ def __init__(self,
33
+ unquantized_modules: Optional[list[str]] = None) -> None:
34
super().__init__()
35
+ self.unquantized_modules = unquantized_modules or []
36
37
def __repr__(self) -> str:
38
return ("GGUFConfig()")
@@ -55,6 +58,8 @@ class GGUFConfig(QuantizationConfig):
58
def get_quant_method(self, layer: torch.nn.Module,
59
prefix: str) -> Optional["QuantizeMethodBase"]:
60
if isinstance(layer, LinearBase):
61
+ if is_layer_skipped_gguf(prefix, self.unquantized_modules):
62
+ return UnquantizedLinearMethod()
63
return GGUFLinearMethod(self)
64
elif isinstance(layer, VocabParallelEmbedding):
65
return GGUFEmbeddingMethod(self)
@@ -63,6 +68,10 @@ class GGUFConfig(QuantizationConfig):
68
return None
69
70
71
+def is_layer_skipped_gguf(prefix: str, unquantized_modules: list[str]):
72
+ return any(module_name in prefix for module_name in unquantized_modules)
73
+
74
+
75
UNQUANTIZED_TYPES = {WeightType.F32, WeightType.F16, WeightType.BF16}
76
STANDARD_QUANT_TYPES = {
77
WeightType.Q4_0,
vllm/model_executor/model_loader/gguf_loader.py
+13
-1
@@ -14,7 +14,8 @@ from vllm.model_executor.model_loader.base_loader import BaseModelLoader
14
from vllm.model_executor.model_loader.utils import (
15
initialize_model, process_weights_after_loading, set_default_torch_dtype)
16
from vllm.model_executor.model_loader.weight_utils import (
17
- get_gguf_extra_tensor_names, gguf_quant_weights_iterator)
17
+ get_gguf_extra_tensor_names, get_gguf_weight_type_map,
18
+ gguf_quant_weights_iterator)
19
20
21
class GGUFModelLoader(BaseModelLoader):
@@ -132,6 +133,17 @@ class GGUFModelLoader(BaseModelLoader):
133
local_model_path, gguf_weights_map):
134
model_config.hf_config.update({"tie_word_embeddings": True})
135
136
+ weight_type_map = get_gguf_weight_type_map(model_config.model,
137
+ gguf_weights_map)
138
+
139
+ # filter out unquantized modules to skip
140
+ unquant_names = [
141
+ name.removesuffix(".weight")
142
+ for name, weight_type in weight_type_map.items()
143
+ if weight_type == "F32" and name.endswith(".weight")
144
+ ]
145
+ vllm_config.quant_config.unquantized_modules.extend(unquant_names)
146
+
147
target_device = torch.device(device_config.device)
148
with set_default_torch_dtype(model_config.dtype):
149
with target_device:
vllm/model_executor/model_loader/weight_utils.py
+12
@@ -563,6 +563,18 @@ def get_gguf_extra_tensor_names(
563
return [gguf_to_hf_name_map[key] for key in extra_keys]
564
565
566
+def get_gguf_weight_type_map(
567
+ gguf_file: str, gguf_to_hf_name_map: dict[str, str]) -> dict[str, str]:
568
+ """
569
+ Return GGUF mapped weight's name and its quant type
570
+ """
571
+ reader = gguf.GGUFReader(gguf_file)
572
+ return {
573
+ gguf_to_hf_name_map[tensor.name]: tensor.tensor_type.name
574
+ for tensor in reader.tensors if tensor.name in gguf_to_hf_name_map
575
+ }
576
+
577
+
578
def gguf_quant_weights_iterator(
579
gguf_file: str, gguf_to_hf_name_map: dict[str, str]
580
) -> Generator[tuple[str, torch.Tensor], None, None]: