@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0fa14907da
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[TPU] Add Load-time W8A16 quantization for TPU Backend (#7005)

Siyuan Liu committed Aug 8, 2024 at 18:35 UTC 0fa14907da70f3cbf9eb97d68d760b181a9a25a7
4 files changed +135 -8
vllm/config.py
+6
@@ -244,6 +244,7 @@ class ModelConfig:
244 "fp8", "marlin", "gptq_marlin_24", "gptq_marlin", "awq_marlin",
245 "fbgemm_fp8", "compressed_tensors", "compressed-tensors"
246 ]
247 + tpu_supported_quantization = ["tpu_int8"]
248 if self.quantization is not None:
249 self.quantization = self.quantization.lower()
250
@@ -282,6 +283,11 @@ class ModelConfig:
283 raise ValueError(
284 f"{self.quantization} quantization is currently not "
285 f"supported in ROCm.")
286 + if is_tpu(
287 + ) and self.quantization not in tpu_supported_quantization:
288 + raise ValueError(
289 + f"{self.quantization} quantization is currently not "
290 + f"supported in TPU Backend.")
291 if self.quantization not in optimized_quantization_methods:
292 logger.warning(
293 "%s quantization is not fully "
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -22,11 +22,13 @@ from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
22 from vllm.model_executor.layers.quantization.marlin import MarlinConfig
23 from vllm.model_executor.layers.quantization.qqq import QQQConfig
24 from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
25 +from vllm.model_executor.layers.quantization.tpu_int8 import Int8TpuConfig
26
27 QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
28 "aqlm": AQLMConfig,
29 "awq": AWQConfig,
30 "deepspeedfp": DeepSpeedFPConfig,
31 + "tpu_int8": Int8TpuConfig,
32 "fp8": Fp8Config,
33 "fbgemm_fp8": FBGEMMFp8Config,
34 # The order of gptq methods is important for config.py iteration over
vllm/model_executor/layers/quantization/tpu_int8.py new
+118
@@ -0,0 +1,118 @@
1 +from typing import Any, Dict, List, Optional, Tuple
2 +
3 +import torch
4 +from torch.nn import Module
5 +from torch.nn.parameter import Parameter
6 +
7 +from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8 +from vllm.model_executor.layers.quantization.base_config import (
9 + QuantizationConfig)
10 +from vllm.model_executor.utils import set_weight_attrs
11 +
12 +ACTIVATION_SCHEMES = ["none"]
13 +
14 +
15 +class Int8TpuConfig(QuantizationConfig):
16 + """Int8 Quantization Config class for TPU Backend."""
17 +
18 + def __init__(
19 + self,
20 + activation_scheme: str = "none",
21 + ) -> None:
22 + if activation_scheme not in ACTIVATION_SCHEMES:
23 + raise ValueError(
24 + f"Unsupported activation scheme {activation_scheme}")
25 + self.activation_scheme = activation_scheme
26 +
27 + def get_name(self) -> str:
28 + return "tpu_int8"
29 +
30 + def get_supported_act_dtypes(self) -> List[torch.dtype]:
31 + return [torch.float16, torch.bfloat16]
32 +
33 + @classmethod
34 + def get_min_capability(cls) -> int:
35 + raise NotImplementedError(
36 + "This function should not be called with TPU Backend")
37 +
38 + @staticmethod
39 + def get_config_filenames() -> List[str]:
40 + return []
41 +
42 + @classmethod
43 + def from_config(cls, config: Dict[str, Any]) -> "Int8TpuConfig":
44 + activation_scheme = cls.get_from_keys(config, ["activation_scheme"])
45 + return cls(activation_scheme=activation_scheme)
46 +
47 + def get_quant_method(self, layer: Module,
48 + prefix: str) -> Optional["TPUInt8LinearMethod"]:
49 + if isinstance(layer, LinearBase):
50 + return TPUInt8LinearMethod(self)
51 + return None
52 +
53 + def get_scaled_act_names(self) -> List[str]:
54 + return []
55 +
56 +
57 +class TPUInt8LinearMethod(LinearMethodBase):
58 + """Int8 Linear method for TPU Quant. """
59 +
60 + def __init__(self, quant_config: Int8TpuConfig):
61 + self.quant_config = quant_config
62 +
63 + def create_weights(self, layer: Module, input_size_per_partition: int,
64 + output_partition_sizes: List[int], input_size: int,
65 + output_size: int, params_dtype: torch.dtype,
66 + **extra_weight_attrs):
67 + weight = Parameter(torch.empty(sum(output_partition_sizes),
68 + input_size_per_partition,
69 + dtype=params_dtype),
70 + requires_grad=False)
71 + layer.register_parameter("weight", weight)
72 + set_weight_attrs(weight, {
73 + **extra_weight_attrs,
74 + "input_dim": 1,
75 + "output_dim": 0,
76 + })
77 +
78 + def _quantize_weight(
79 + self, weight: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
80 + weight_dtype = weight.dtype
81 + weight = weight.cpu().to(torch.float32)
82 + n_bit = 8
83 + eps = 1e-5
84 + max_int = 2**(n_bit - 1) - 1
85 + min_int = -(2**(n_bit - 1))
86 + max_val = weight.abs().amax(dim=-1, keepdim=True)
87 + max_val = max_val.clamp(min=eps)
88 + qscale = max_val / max_int
89 + qweight = torch.clamp(torch.round(weight * (1.0 / qscale)), min_int,
90 + max_int).to(torch.int8)
91 + qscale = qscale.squeeze().to(weight_dtype)
92 + return qweight, qscale
93 +
94 + def process_weights_after_loading(self, layer: Module) -> None:
95 + device = layer.weight.device
96 + qweight, qscale = self._quantize_weight(layer.weight)
97 + qweight = qweight.to(device)
98 + qscale = qscale.to(device)
99 + layer.weight = Parameter(qweight, requires_grad=False)
100 + layer.scale = Parameter(qscale, requires_grad=False)
101 +
102 + def apply(self,
103 + layer: torch.nn.Module,
104 + x: torch.Tensor,
105 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
106 + try:
107 + import torch_xla.experimental.xla_quantized_matmul # noqa: F401
108 + except ImportError as err:
109 + raise ImportError(
110 + "Please install torch_xla by following the instructions at "
111 + "https://docs.vllm.ai/en/latest/getting_started/tpu-installation.html " # noqa: E501
112 + "to run vLLM on TPU.") from err
113 + weight = layer.weight
114 + scale = layer.scale
115 + out = torch.ops.xla.quantized_matmul(x, weight, scale)
116 + if bias is not None:
117 + out = out + bias
118 + return out
vllm/model_executor/model_loader/loader.py
+9 -8
@@ -94,14 +94,15 @@ def _get_quantization_config(
94 """Get the quantization config."""
95 if model_config.quantization is not None:
96 quant_config = get_quant_config(model_config, load_config)
97 - capability = current_platform.get_device_capability()
98 - capability = capability[0] * 10 + capability[1]
99 - if capability < quant_config.get_min_capability():
100 - raise ValueError(
101 - f"The quantization method {model_config.quantization} is not "
102 - "supported for the current GPU. "
103 - f"Minimum capability: {quant_config.get_min_capability()}. "
104 - f"Current capability: {capability}.")
97 + if not is_tpu():
98 + capability = current_platform.get_device_capability()
99 + capability = capability[0] * 10 + capability[1]
100 + if capability < quant_config.get_min_capability():
101 + raise ValueError(
102 + f"The quantization method {model_config.quantization} "
103 + "is not supported for the current GPU. "
104 + f"Minimum capability: {quant_config.get_min_capability()}. "
105 + f"Current capability: {capability}.")
106 supported_dtypes = quant_config.get_supported_act_dtypes()
107 if model_config.dtype not in supported_dtypes:
108 raise ValueError(