A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Misc] Support register quantization method out-of-tree (#11969)
yancong committed
Jan 19, 2025 at 08:13 UTC
32eb0da808ea162a2d6758ff0bd9bdd0934b5fd5
2 files changed
+158
tests/quantization/test_register_quantization_config.py
new
+117
@@ -0,0 +1,117 @@
1
+"""Tests register custom quantization config.
2
+
3
+See https://github.com/vllm-project/vllm/issues/11926 for more details.
4
+
5
+Run `pytest tests/quantization/test_register_quantization_config.py`.
6
+"""
7
+from typing import Any, Dict, List, Optional
8
+
9
+import pytest
10
+import torch
11
+import torch.nn.functional as F
12
+
13
+from vllm.model_executor.layers.linear import LinearBase # noqa: E501
14
+from vllm.model_executor.layers.linear import UnquantizedLinearMethod
15
+from vllm.model_executor.layers.quantization import (
16
+ get_quantization_config, register_quantization_config)
17
+from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
18
+ QuantizationConfig)
19
+
20
+
21
+class FakeQuantLinearMethod(UnquantizedLinearMethod):
22
+ """Fake quantization linear method for per-token dynamic quantization."""
23
+
24
+ def __init__(self, num_bits: int = 8) -> None:
25
+ """Initialize the quantization method."""
26
+ super().__init__()
27
+ self.num_bits = num_bits
28
+
29
+ def apply(self,
30
+ layer: "torch.nn.Module",
31
+ x: "torch.Tensor",
32
+ bias: Optional["torch.Tensor"] = None) -> "torch.Tensor":
33
+ """Perform fake quantization before the linear layer."""
34
+
35
+ # Calculate the scales dynamically
36
+ max_val = torch.amax(x, dim=(0, -1), keepdims=True)
37
+ min_val = torch.amin(x, dim=(0, -1), keepdims=True)
38
+ scales = (max_val - min_val) / (2**self.num_bits - 1)
39
+
40
+ # Fake quantize the input
41
+ quant_x = torch.clamp(torch.round(x / scales), -2**(self.num_bits - 1),
42
+ 2**(self.num_bits - 1) - 1)
43
+ dequant_x = quant_x * scales
44
+
45
+ return F.linear(dequant_x, layer.weight, bias)
46
+
47
+
48
+@register_quantization_config("custom_quant")
49
+class CustomQuantConfig(QuantizationConfig):
50
+ """Custom quantization config for per-token dynamic fake quantization."""
51
+
52
+ def __init__(self, num_bits: int = 8) -> None:
53
+ """Initialize the quantization config."""
54
+ self.num_bits = num_bits
55
+
56
+ def get_name(self) -> str:
57
+ """Name of the quantization method."""
58
+ return "custom_quant"
59
+
60
+ def get_supported_act_dtypes(self) -> List["torch.dtype"]:
61
+ """List of supported activation dtypes."""
62
+ return [torch.float16, torch.bfloat16]
63
+
64
+ @classmethod
65
+ def get_min_capability(cls) -> int:
66
+ """Minimum GPU capability to support the quantization method."""
67
+ return -1
68
+
69
+ @staticmethod
70
+ def get_config_filenames() -> List[str]:
71
+ """List of filenames to search for in the model directory."""
72
+ return []
73
+
74
+ @classmethod
75
+ def from_config(cls, config: Dict[str, Any]) -> "CustomQuantConfig":
76
+ """Create a config class from the model's quantization config."""
77
+ return CustomQuantConfig(num_bits=config.get("num_bits", 8))
78
+
79
+ def get_quant_method(self, layer: "torch.nn.Module",
80
+ prefix: str) -> Optional["FakeQuantLinearMethod"]:
81
+ """Get the quantize method to use for the quantized layer."""
82
+ if isinstance(layer, LinearBase):
83
+ return FakeQuantLinearMethod(num_bits=self.num_bits)
84
+ return None
85
+
86
+
87
+def test_register_quantization_config():
88
+ """Test register custom quantization config."""
89
+
90
+ # The quantization method `custom_quant` should be registered.
91
+ assert get_quantization_config("custom_quant") == CustomQuantConfig
92
+
93
+ # The quantization method `custom_quant` is already exists,
94
+ # should raise an error.
95
+ with pytest.raises(ValueError):
96
+ register_quantization_config("custom_quant")(CustomQuantConfig)
97
+
98
+
99
+@pytest.mark.parametrize(argnames="model",
100
+ argvalues=[
101
+ "meta-llama/Meta-Llama-3-8B-Instruct",
102
+ ])
103
+def test_custom_quant(vllm_runner, model):
104
+ """Test infer with the custom quantization method."""
105
+ with vllm_runner(model_name=model,
106
+ quantization="custom_quant",
107
+ enforce_eager=True) as llm:
108
+
109
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
110
+ layer = model.model.layers[0]
111
+ qkv_proj = layer.self_attn.qkv_proj
112
+
113
+ # Check the quantization method is FakeQuantLinearMethod
114
+ assert isinstance(qkv_proj.quant_method, FakeQuantLinearMethod)
115
+
116
+ output = llm.generate_greedy("Hello my name is", max_tokens=20)
117
+ assert output
vllm/model_executor/layers/quantization/__init__.py
+41
@@ -29,6 +29,45 @@ QUANTIZATION_METHODS: List[str] = [
29
"quark"
30
]
31
32
+# The customized quantization methods which will be added to this dict.
33
+_CUSTOMIZED_METHOD_TO_QUANT_CONFIG = {}
34
+
35
+
36
+def register_quantization_config(quantization: str):
37
+ """Register a customized vllm quantization config.
38
+
39
+ When a quantization method is not supported by vllm, you can register a customized
40
+ quantization config to support it.
41
+
42
+ Args:
43
+ quantization (str): The quantization method name.
44
+
45
+ Examples:
46
+ >>> from vllm.model_executor.layers.quantization import register_quantization_config
47
+ >>> from vllm.model_executor.layers.quantization import get_quantization_config
48
+ >>> from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
49
+ >>>
50
+ >>> @register_quantization_config("my_quant")
51
+ ... class MyQuantConfig(QuantizationConfig):
52
+ ... pass
53
+ >>>
54
+ >>> get_quantization_config("my_quant")
55
+ <class 'MyQuantConfig'>
56
+ """ # noqa: E501
57
+
58
+ def _wrapper(quant_config_cls):
59
+ if quantization in QUANTIZATION_METHODS:
60
+ raise ValueError(
61
+ f"The quantization method `{quantization}` is already exists.")
62
+ if not issubclass(quant_config_cls, QuantizationConfig):
63
+ raise ValueError("The quantization config must be a subclass of "
64
+ "`QuantizationConfig`.")
65
+ _CUSTOMIZED_METHOD_TO_QUANT_CONFIG[quantization] = quant_config_cls
66
+ QUANTIZATION_METHODS.append(quantization)
67
+ return quant_config_cls
68
+
69
+ return _wrapper
70
+
71
72
def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
73
if quantization not in QUANTIZATION_METHODS:
@@ -84,6 +123,8 @@ def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
123
"ipex": IPEXConfig,
124
"quark": QuarkConfig
125
}
126
+ # Update the `method_to_config` with customized quantization methods.
127
+ method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG)
128
129
return method_to_config[quantization]
130