A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[TPU] Add Load-time W8A16 quantization for TPU Backend (#7005)
Siyuan Liu committed
Aug 8, 2024 at 18:35 UTC
0fa14907da70f3cbf9eb97d68d760b181a9a25a7
4 files changed
+135
-8
vllm/config.py
+6
@@ -244,6 +244,7 @@ class ModelConfig:
244
"fp8", "marlin", "gptq_marlin_24", "gptq_marlin", "awq_marlin",
245
"fbgemm_fp8", "compressed_tensors", "compressed-tensors"
246
]
247
+ tpu_supported_quantization = ["tpu_int8"]
248
if self.quantization is not None:
249
self.quantization = self.quantization.lower()
250
@@ -282,6 +283,11 @@ class ModelConfig:
283
raise ValueError(
284
f"{self.quantization} quantization is currently not "
285
f"supported in ROCm.")
286
+ if is_tpu(
287
+ ) and self.quantization not in tpu_supported_quantization:
288
+ raise ValueError(
289
+ f"{self.quantization} quantization is currently not "
290
+ f"supported in TPU Backend.")
291
if self.quantization not in optimized_quantization_methods:
292
logger.warning(
293
"%s quantization is not fully "
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -22,11 +22,13 @@ from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
22
from vllm.model_executor.layers.quantization.marlin import MarlinConfig
23
from vllm.model_executor.layers.quantization.qqq import QQQConfig
24
from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
25
+from vllm.model_executor.layers.quantization.tpu_int8 import Int8TpuConfig
26
27
QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
28
"aqlm": AQLMConfig,
29
"awq": AWQConfig,
30
"deepspeedfp": DeepSpeedFPConfig,
31
+ "tpu_int8": Int8TpuConfig,
32
"fp8": Fp8Config,
33
"fbgemm_fp8": FBGEMMFp8Config,
34
# The order of gptq methods is important for config.py iteration over
vllm/model_executor/layers/quantization/tpu_int8.py
new
+118
@@ -0,0 +1,118 @@
1
+from typing import Any, Dict, List, Optional, Tuple
2
+
3
+import torch
4
+from torch.nn import Module
5
+from torch.nn.parameter import Parameter
6
+
7
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8
+from vllm.model_executor.layers.quantization.base_config import (
9
+ QuantizationConfig)
10
+from vllm.model_executor.utils import set_weight_attrs
11
+
12
+ACTIVATION_SCHEMES = ["none"]
13
+
14
+
15
+class Int8TpuConfig(QuantizationConfig):
16
+ """Int8 Quantization Config class for TPU Backend."""
17
+
18
+ def __init__(
19
+ self,
20
+ activation_scheme: str = "none",
21
+ ) -> None:
22
+ if activation_scheme not in ACTIVATION_SCHEMES:
23
+ raise ValueError(
24
+ f"Unsupported activation scheme {activation_scheme}")
25
+ self.activation_scheme = activation_scheme
26
+
27
+ def get_name(self) -> str:
28
+ return "tpu_int8"
29
+
30
+ def get_supported_act_dtypes(self) -> List[torch.dtype]:
31
+ return [torch.float16, torch.bfloat16]
32
+
33
+ @classmethod
34
+ def get_min_capability(cls) -> int:
35
+ raise NotImplementedError(
36
+ "This function should not be called with TPU Backend")
37
+
38
+ @staticmethod
39
+ def get_config_filenames() -> List[str]:
40
+ return []
41
+
42
+ @classmethod
43
+ def from_config(cls, config: Dict[str, Any]) -> "Int8TpuConfig":
44
+ activation_scheme = cls.get_from_keys(config, ["activation_scheme"])
45
+ return cls(activation_scheme=activation_scheme)
46
+
47
+ def get_quant_method(self, layer: Module,
48
+ prefix: str) -> Optional["TPUInt8LinearMethod"]:
49
+ if isinstance(layer, LinearBase):
50
+ return TPUInt8LinearMethod(self)
51
+ return None
52
+
53
+ def get_scaled_act_names(self) -> List[str]:
54
+ return []
55
+
56
+
57
+class TPUInt8LinearMethod(LinearMethodBase):
58
+ """Int8 Linear method for TPU Quant. """
59
+
60
+ def __init__(self, quant_config: Int8TpuConfig):
61
+ self.quant_config = quant_config
62
+
63
+ def create_weights(self, layer: Module, input_size_per_partition: int,
64
+ output_partition_sizes: List[int], input_size: int,
65
+ output_size: int, params_dtype: torch.dtype,
66
+ **extra_weight_attrs):
67
+ weight = Parameter(torch.empty(sum(output_partition_sizes),
68
+ input_size_per_partition,
69
+ dtype=params_dtype),
70
+ requires_grad=False)
71
+ layer.register_parameter("weight", weight)
72
+ set_weight_attrs(weight, {
73
+ **extra_weight_attrs,
74
+ "input_dim": 1,
75
+ "output_dim": 0,
76
+ })
77
+
78
+ def _quantize_weight(
79
+ self, weight: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
80
+ weight_dtype = weight.dtype
81
+ weight = weight.cpu().to(torch.float32)
82
+ n_bit = 8
83
+ eps = 1e-5
84
+ max_int = 2**(n_bit - 1) - 1
85
+ min_int = -(2**(n_bit - 1))
86
+ max_val = weight.abs().amax(dim=-1, keepdim=True)
87
+ max_val = max_val.clamp(min=eps)
88
+ qscale = max_val / max_int
89
+ qweight = torch.clamp(torch.round(weight * (1.0 / qscale)), min_int,
90
+ max_int).to(torch.int8)
91
+ qscale = qscale.squeeze().to(weight_dtype)
92
+ return qweight, qscale
93
+
94
+ def process_weights_after_loading(self, layer: Module) -> None:
95
+ device = layer.weight.device
96
+ qweight, qscale = self._quantize_weight(layer.weight)
97
+ qweight = qweight.to(device)
98
+ qscale = qscale.to(device)
99
+ layer.weight = Parameter(qweight, requires_grad=False)
100
+ layer.scale = Parameter(qscale, requires_grad=False)
101
+
102
+ def apply(self,
103
+ layer: torch.nn.Module,
104
+ x: torch.Tensor,
105
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
106
+ try:
107
+ import torch_xla.experimental.xla_quantized_matmul # noqa: F401
108
+ except ImportError as err:
109
+ raise ImportError(
110
+ "Please install torch_xla by following the instructions at "
111
+ "https://docs.vllm.ai/en/latest/getting_started/tpu-installation.html " # noqa: E501
112
+ "to run vLLM on TPU.") from err
113
+ weight = layer.weight
114
+ scale = layer.scale
115
+ out = torch.ops.xla.quantized_matmul(x, weight, scale)
116
+ if bias is not None:
117
+ out = out + bias
118
+ return out
vllm/model_executor/model_loader/loader.py
+9
-8
@@ -94,14 +94,15 @@ def _get_quantization_config(
94
"""Get the quantization config."""
95
if model_config.quantization is not None:
96
quant_config = get_quant_config(model_config, load_config)
97
- capability = current_platform.get_device_capability()
98
- capability = capability[0] * 10 + capability[1]
99
- if capability < quant_config.get_min_capability():
100
- raise ValueError(
101
- f"The quantization method {model_config.quantization} is not "
102
- "supported for the current GPU. "
103
- f"Minimum capability: {quant_config.get_min_capability()}. "
104
- f"Current capability: {capability}.")
97
+ if not is_tpu():
98
+ capability = current_platform.get_device_capability()
99
+ capability = capability[0] * 10 + capability[1]
100
+ if capability < quant_config.get_min_capability():
101
+ raise ValueError(
102
+ f"The quantization method {model_config.quantization} "
103
+ "is not supported for the current GPU. "
104
+ f"Minimum capability: {quant_config.get_min_capability()}. "
105
+ f"Current capability: {capability}.")
106
supported_dtypes = quant_config.get_supported_act_dtypes()
107
if model_config.dtype not in supported_dtypes:
108
raise ValueError(