A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[TPU][Quantization] TPU `W8A8` (#11785)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Robert Shaw committed
Jan 8, 2025 at 14:33 UTC
56fe4c297c7d9d872eccc19e3edbf1d75e1a30e2
18 files changed
+565
-190
.buildkite/run-tpu-test.sh
+10
-1
@@ -14,4 +14,13 @@ remove_docker_container
14
# For HF_TOKEN.
15
source /etc/environment
16
# Run a simple end-to-end example.
17
-docker run --privileged --net host --shm-size=16G -it -e "HF_TOKEN=$HF_TOKEN" --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && python3 -m pip install lm_eval[api]==0.4.4 && pytest -v -s /workspace/vllm/tests/entrypoints/openai/test_accuracy.py && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference/offline_inference_tpu.py"
17
+docker run --privileged --net host --shm-size=16G -it \
18
+ -e "HF_TOKEN=$HF_TOKEN" --name tpu-test \
19
+ vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git \
20
+ && python3 -m pip install pytest \
21
+ && python3 -m pip install lm_eval[api]==0.4.4 \
22
+ && pytest -v -s /workspace/vllm/tests/entrypoints/openai/test_accuracy.py \
23
+ && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py \
24
+ && python3 /workspace/vllm/tests/tpu/test_compilation.py \
25
+ && python3 /workspace/vllm/tests/tpu/test_quantization_accuracy.py \
26
+ && python3 /workspace/vllm/examples/offline_inference/offline_inference_tpu.py"
tests/tpu/test_quantization_accuracy.py
new
+49
@@ -0,0 +1,49 @@
1
+from dataclasses import dataclass
2
+
3
+import lm_eval
4
+import pytest
5
+
6
+TASK = "gsm8k"
7
+FILTER = "exact_match,strict-match"
8
+RTOL = 0.03
9
+
10
+
11
+@dataclass
12
+class GSM8KAccuracyTestConfig:
13
+ model_name: str
14
+ excepted_value: float
15
+
16
+ def get_model_args(self) -> str:
17
+ return (f"pretrained={self.model_name},"
18
+ "max_model_len=4096,max_num_seqs=32")
19
+
20
+
21
+# NOTE: Accuracy scores measured on GPUs.
22
+ACCURACY_CONFIGS = [
23
+ GSM8KAccuracyTestConfig(
24
+ model_name="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
25
+ excepted_value=0.76), # no bias
26
+ # NOTE(rob): We cannot re-initialize VLLM in the same process for TPU,
27
+ # so only one of these tests can run in a single call to pytest. As
28
+ # a follow up, move this into the LM-EVAL section of the CI.
29
+ # GSM8KAccuracyTestConfig(
30
+ # model_name="neuralmagic/Qwen2-7B-Instruct-quantized.w8a8",
31
+ # excepted_value=0.66), # bias in QKV layers
32
+]
33
+
34
+
35
+@pytest.mark.parametrize("config", ACCURACY_CONFIGS)
36
+def test_gsm8k_correctness(config: GSM8KAccuracyTestConfig):
37
+
38
+ results = lm_eval.simple_evaluate(
39
+ model="vllm",
40
+ model_args=config.get_model_args(),
41
+ tasks="gsm8k",
42
+ batch_size="auto",
43
+ )
44
+
45
+ EXPECTED_VALUE = config.excepted_value
46
+ measured_value = results["results"][TASK][FILTER]
47
+ assert (measured_value - RTOL < EXPECTED_VALUE
48
+ and measured_value + RTOL > EXPECTED_VALUE
49
+ ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+31
-74
@@ -1,14 +1,13 @@
1
-from typing import Callable, List, Optional
1
+from typing import Callable, List, Optional, Set
2
3
import torch
4
from compressed_tensors.quantization import QuantizationStrategy
5
-from torch.nn import Parameter
5
6
from vllm.logger import init_logger
7
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
CompressedTensorsScheme)
10
-from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11
- apply_int8_linear, convert_to_channelwise)
9
+from vllm.model_executor.layers.quantization.kernels.scaled_mm import (
10
+ ScaledMMLinearLayerConfig, choose_scaled_mm_linear_kernel)
11
from vllm.model_executor.parameter import (BasevLLMParameter,
12
ChannelQuantScaleParameter,
13
ModelWeightParameter,
@@ -18,6 +17,7 @@ logger = init_logger(__name__)
17
18
19
class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
20
+ _kernel_backends_being_used: Set[str] = set()
21
22
def __init__(self, strategy: str, is_static_input_scheme: bool,
23
input_symmetric: bool):
@@ -30,74 +30,25 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
30
# turing and up
31
return 75
32
33
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
34
- # WEIGHT
35
- # Cutlass kernels need transposed weight.
36
- weight = layer.weight
37
- layer.weight = Parameter(weight.t(), requires_grad=False)
38
-
39
- # WEIGHT SCALE
40
- # Cutlass kernels support only per-tensor and per-channel.
41
- # If we have a fused module (QKV, MLP) with per tensor scales (thus N
42
- # scales being passed to the kernel), convert to the per-channel case.
43
- is_fused_module = len(self.logical_widths) > 1
44
- if is_fused_module and self.strategy == QuantizationStrategy.TENSOR:
45
- ws_channelwise = convert_to_channelwise(layer.weight_scale,
46
- self.logical_widths)
47
- layer.weight_scale = Parameter(ws_channelwise, requires_grad=False)
48
- else:
49
- layer.weight_scale = Parameter(layer.weight_scale.data,
50
- requires_grad=False)
51
- # INPUT SCALE
52
- if self.is_static_input_scheme:
53
- if self.input_symmetric:
54
- layer.input_scale = Parameter(layer.input_scale.max(),
55
- requires_grad=False)
56
- layer.input_zero_point = None
57
- else:
58
- # reconstruct the ranges
59
- int8_traits = torch.iinfo(torch.int8)
60
- azps = layer.input_zero_point.to(dtype=torch.int32)
61
- range_max = (layer.input_scale *
62
- (int8_traits.max - azps)).max()
63
- range_min = (layer.input_scale *
64
- (int8_traits.min - azps)).min()
65
-
66
- scale = (range_max - range_min) / (int8_traits.max -
67
- int8_traits.min)
68
- layer.input_scale = Parameter(scale, requires_grad=False)
69
-
70
- # AZP loaded as int8 but used as int32
71
- azp = (int8_traits.min -
72
- range_min / scale).to(dtype=torch.int32)
73
- layer.input_zero_point = Parameter(azp, requires_grad=False)
74
-
75
- else:
76
- layer.input_scale = None
77
- layer.input_zero_point = None
78
-
79
- # azp_adj is the AZP adjustment term, used to account for weights.
80
- # It does not depend on scales or azp, so it is the same for
81
- # static and dynamic quantization.
82
- # For more details, see csrc/quantization/cutlass_w8a8/Epilogues.md
83
- # https://github.com/vllm-project/vllm/blob/8d59dbb00044a588cab96bcdc028006ed922eb06/csrc/quantization/cutlass_w8a8/Epilogues.md
84
- if not self.input_symmetric:
85
- azp_adj = layer.weight.sum(dim=0, keepdim=True, dtype=torch.int32)
86
- if self.is_static_input_scheme:
87
- # cutlass_w8a8 requires azp to be folded into azp_adj
88
- # in the per-tensor case
89
- azp_adj = layer.input_zero_point * azp_adj
90
-
91
- layer.azp_adj = azp_adj
92
- else:
93
- layer.azp_adj = None
94
-
33
def create_weights(self, layer: torch.nn.Module,
34
output_partition_sizes: List[int],
35
input_size_per_partition: int,
36
params_dtype: torch.dtype, weight_loader: Callable,
37
**kwargs):
100
- self.logical_widths = output_partition_sizes
38
+ layer.logical_widths = output_partition_sizes
39
+
40
+ scaled_mm_linear_kernel_config = ScaledMMLinearLayerConfig(
41
+ is_channelwise=(self.strategy == QuantizationStrategy.CHANNEL),
42
+ is_static_input_scheme=self.is_static_input_scheme,
43
+ input_symmetric=self.input_symmetric)
44
+
45
+ kernel_type = choose_scaled_mm_linear_kernel(
46
+ scaled_mm_linear_kernel_config)
47
+
48
+ if kernel_type.__name__ not in self._kernel_backends_being_used:
49
+ logger.info("Using %s for CompressedTensorsW8A8Int8",
50
+ kernel_type.__name__)
51
+ self._kernel_backends_being_used.add(kernel_type.__name__)
52
53
# WEIGHT
54
weight = ModelWeightParameter(data=torch.empty(
@@ -140,12 +91,18 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
91
weight_loader=weight_loader)
92
layer.register_parameter("input_zero_point", input_zero_point)
93
94
+ self.kernel = kernel_type(c=scaled_mm_linear_kernel_config,
95
+ w_q_param_name="weight",
96
+ w_s_param_name="weight_scale",
97
+ i_s_param_name="input_scale",
98
+ i_zp_param_name="input_zero_point",
99
+ azp_adj_param_name="azp_adj")
100
+
101
+ # Checkpoints are serialized in compressed-tensors format, which is
102
+ # different from the format the kernel may want. Handle repacking here.
103
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
104
+ self.kernel.process_weights_after_loading(layer)
105
+
106
def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
107
bias: Optional[torch.Tensor]) -> torch.Tensor:
145
- return apply_int8_linear(input=x,
146
- weight=layer.weight,
147
- weight_scale=layer.weight_scale,
148
- input_scale=layer.input_scale,
149
- input_zero_point=layer.input_zero_point,
150
- azp_adj=layer.azp_adj,
151
- bias=bias)
108
+ return self.kernel.apply_weights(layer, x, bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+1
-1
@@ -6,7 +6,7 @@ from compressed_tensors.quantization import ActivationOrdering
6
from vllm.logger import init_logger
7
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8
CompressedTensorsScheme)
9
-from vllm.model_executor.layers.quantization.kernels import (
9
+from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
10
MPLinearLayerConfig, choose_mp_linear_kernel)
11
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12
marlin_repeat_scales_on_all_ranks)
vllm/model_executor/layers/quantization/gptq_marlin.py
+1
-1
@@ -11,7 +11,7 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11
set_weight_attrs)
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig)
14
-from vllm.model_executor.layers.quantization.kernels import (
14
+from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
15
MPLinearLayerConfig, choose_mp_linear_kernel)
16
from vllm.model_executor.layers.quantization.utils import replace_parameter
17
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
vllm/model_executor/layers/quantization/kernels/__init__.py
-74
@@ -1,74 +0,0 @@
1
-from typing import List, Optional, Type
2
-
3
-import vllm.envs as envs
4
-from vllm.model_executor.layers.quantization.kernels.exllama import (
5
- ExllamaLinearKernel)
6
-from vllm.model_executor.layers.quantization.kernels.machete import (
7
- MacheteLinearKernel)
8
-from vllm.model_executor.layers.quantization.kernels.marlin import (
9
- MarlinLinearKernel)
10
-from vllm.model_executor.layers.quantization.kernels.MPLinearKernel import (
11
- MPLinearKernel, MPLinearLayerConfig)
12
-from vllm.platforms import current_platform
13
-
14
-# in priority/performance order (when available)
15
-_POSSIBLE_KERNELS: List[Type[MPLinearKernel]] = [
16
- MacheteLinearKernel,
17
- MarlinLinearKernel,
18
- ExllamaLinearKernel,
19
-]
20
-
21
-
22
-def choose_mp_linear_kernel(
23
- config: MPLinearLayerConfig,
24
- compute_capability: Optional[int] = None) -> Type[MPLinearKernel]:
25
- """
26
- Choose an MPLinearKernel that can implement the given config for the given
27
- compute capability. Attempts to choose the best kernel in terms of
28
- performance.
29
-
30
- Args:
31
- config (MPLinearLayerConfig): Description of the linear layer to be
32
- implemented.
33
- compute_capability (Optional[int], optional): The compute capability of
34
- the target device, if None uses `current_platform` to get the compute
35
- capability. Defaults to None.
36
-
37
- Raises:
38
- ValueError: If no kernel can implement the given config.
39
-
40
- Returns:
41
- Type[MPLinearKernel]: Chosen kernel.
42
- """
43
- if compute_capability is None:
44
- if current_platform is None:
45
- raise ValueError("Cannot determine compute capability")
46
- _cc = current_platform.get_device_capability()
47
- compute_capability = _cc[0] * 10 + _cc[1]
48
-
49
- failure_reasons = []
50
- for kernel in _POSSIBLE_KERNELS:
51
- if kernel.__name__ in envs.VLLM_DISABLED_KERNELS:
52
- failure_reasons.append(
53
- f' {kernel.__name__} disabled by environment variable')
54
- continue
55
-
56
- if kernel.get_min_capability() > compute_capability:
57
- failure_reasons.append(
58
- f"{kernel.__name__} requires capability "
59
- f"{kernel.get_min_capability()}, current compute capability "
60
- f"is {compute_capability}")
61
- continue
62
-
63
- can_implement, failure_reason = kernel.can_implement(config)
64
- if can_implement:
65
- return kernel
66
- else:
67
- failure_reasons.append(
68
- f' {kernel.__name__} cannot implement due to: {failure_reason}'
69
- )
70
-
71
- raise ValueError(
72
- "Failed to find a kernel that can implement the "\
73
- "WNA16 linear layer. Reasons: \n"
74
- + '\n'.join(failure_reasons))
vllm/model_executor/layers/quantization/kernels/mixed_precision/MPLinearKernel.py
renamed
vllm/model_executor/layers/quantization/kernels/mixed_precision/__init__.py
new
+74
@@ -0,0 +1,74 @@
1
+from typing import List, Optional, Type
2
+
3
+import vllm.envs as envs
4
+from vllm.model_executor.layers.quantization.kernels.mixed_precision.exllama import ( # noqa: E501
5
+ ExllamaLinearKernel)
6
+from vllm.model_executor.layers.quantization.kernels.mixed_precision.machete import ( # noqa: E501
7
+ MacheteLinearKernel)
8
+from vllm.model_executor.layers.quantization.kernels.mixed_precision.marlin import ( # noqa: E501
9
+ MarlinLinearKernel)
10
+from vllm.model_executor.layers.quantization.kernels.mixed_precision.MPLinearKernel import ( # noqa: E501
11
+ MPLinearKernel, MPLinearLayerConfig)
12
+from vllm.platforms import current_platform
13
+
14
+# in priority/performance order (when available)
15
+_POSSIBLE_KERNELS: List[Type[MPLinearKernel]] = [
16
+ MacheteLinearKernel,
17
+ MarlinLinearKernel,
18
+ ExllamaLinearKernel,
19
+]
20
+
21
+
22
+def choose_mp_linear_kernel(
23
+ config: MPLinearLayerConfig,
24
+ compute_capability: Optional[int] = None) -> Type[MPLinearKernel]:
25
+ """
26
+ Choose an MPLinearKernel that can implement the given config for the given
27
+ compute capability. Attempts to choose the best kernel in terms of
28
+ performance.
29
+
30
+ Args:
31
+ config (MPLinearLayerConfig): Description of the linear layer to be
32
+ implemented.
33
+ compute_capability (Optional[int], optional): The compute capability of
34
+ the target device, if None uses `current_platform` to get the compute
35
+ capability. Defaults to None.
36
+
37
+ Raises:
38
+ ValueError: If no kernel can implement the given config.
39
+
40
+ Returns:
41
+ Type[MPLinearKernel]: Chosen kernel.
42
+ """
43
+ if compute_capability is None:
44
+ if current_platform is None:
45
+ raise ValueError("Cannot determine compute capability")
46
+ _cc = current_platform.get_device_capability()
47
+ compute_capability = _cc[0] * 10 + _cc[1]
48
+
49
+ failure_reasons = []
50
+ for kernel in _POSSIBLE_KERNELS:
51
+ if kernel.__name__ in envs.VLLM_DISABLED_KERNELS:
52
+ failure_reasons.append(
53
+ f' {kernel.__name__} disabled by environment variable')
54
+ continue
55
+
56
+ if kernel.get_min_capability() > compute_capability:
57
+ failure_reasons.append(
58
+ f"{kernel.__name__} requires capability "
59
+ f"{kernel.get_min_capability()}, current compute capability "
60
+ f"is {compute_capability}")
61
+ continue
62
+
63
+ can_implement, failure_reason = kernel.can_implement(config)
64
+ if can_implement:
65
+ return kernel
66
+ else:
67
+ failure_reasons.append(
68
+ f' {kernel.__name__} cannot implement due to: {failure_reason}'
69
+ )
70
+
71
+ raise ValueError(
72
+ "Failed to find a kernel that can implement the "\
73
+ "WNA16 linear layer. Reasons: \n"
74
+ + '\n'.join(failure_reasons))
vllm/model_executor/layers/quantization/kernels/mixed_precision/exllama.py
renamed
vllm/model_executor/layers/quantization/kernels/mixed_precision/machete.py
renamed
vllm/model_executor/layers/quantization/kernels/mixed_precision/marlin.py
renamed
vllm/model_executor/layers/quantization/kernels/scaled_mm/ScaledMMLinearKernel.py
new
+64
@@ -0,0 +1,64 @@
1
+from abc import ABC, abstractmethod
2
+from dataclasses import dataclass
3
+from typing import Optional, Tuple
4
+
5
+import torch
6
+
7
+
8
+@dataclass
9
+class ScaledMMLinearLayerConfig:
10
+ is_channelwise: bool
11
+ is_static_input_scheme: bool
12
+ input_symmetric: bool
13
+
14
+
15
+class ScaledMMLinearKernel(ABC):
16
+
17
+ @classmethod
18
+ @abstractmethod
19
+ def get_min_capability(cls) -> int:
20
+ raise NotImplementedError
21
+
22
+ @classmethod
23
+ @abstractmethod
24
+ def can_implement(
25
+ cls, c: ScaledMMLinearLayerConfig) -> Tuple[bool, Optional[str]]:
26
+ raise NotImplementedError
27
+
28
+ def __init__(self, c: ScaledMMLinearLayerConfig, w_q_param_name: str,
29
+ w_s_param_name: str, i_s_param_name: str,
30
+ i_zp_param_name: str, azp_adj_param_name: str) -> None:
31
+ assert self.can_implement(c)
32
+ self.config = c
33
+ self.w_q_name = w_q_param_name
34
+ self.w_s_name = w_s_param_name
35
+ self.i_s_name = i_s_param_name
36
+ self.i_zp_name = i_zp_param_name
37
+ self.azp_adj_name = azp_adj_param_name
38
+
39
+ @abstractmethod
40
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
41
+ raise NotImplementedError
42
+
43
+ @abstractmethod
44
+ def apply_weights(self,
45
+ layer: torch.nn.Module,
46
+ x: torch.Tensor,
47
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
48
+ raise NotImplementedError
49
+
50
+ def _get_weight_params(
51
+ self, layer: torch.nn.Module
52
+ ) -> Tuple[torch.Tensor, # weight
53
+ torch.Tensor, # weight_scale
54
+ Optional[torch.Tensor], # input_scale,
55
+ Optional[torch.Tensor], # input_zp
56
+ Optional[torch.Tensor], # azp_adj
57
+ ]:
58
+ return (
59
+ getattr(layer, self.w_q_name),
60
+ getattr(layer, self.w_s_name),
61
+ getattr(layer, self.i_s_name),
62
+ getattr(layer, self.i_zp_name),
63
+ getattr(layer, self.azp_adj_name),
64
+ )
vllm/model_executor/layers/quantization/kernels/scaled_mm/__init__.py
new
+84
@@ -0,0 +1,84 @@
1
+import os
2
+from typing import Dict, List, Optional, Type
3
+
4
+from vllm.model_executor.layers.quantization.kernels.scaled_mm.cutlass import (
5
+ CutlassScaledMMLinearKernel)
6
+from vllm.model_executor.layers.quantization.kernels.scaled_mm.ScaledMMLinearKernel import ( # noqa: E501
7
+ ScaledMMLinearKernel, ScaledMMLinearLayerConfig)
8
+# from vllm.model_executor.layers.quantization.kernels.scaled_mm.triton import (
9
+# TritonScaledMMLinear)
10
+from vllm.model_executor.layers.quantization.kernels.scaled_mm.xla import (
11
+ XLAScaledMMLinearKernel)
12
+from vllm.platforms import PlatformEnum, current_platform
13
+
14
+# in priority/performance order (when available)
15
+_POSSIBLE_KERNELS: Dict[PlatformEnum, List[Type[ScaledMMLinearKernel]]] = {
16
+ PlatformEnum.CPU: [CutlassScaledMMLinearKernel],
17
+ PlatformEnum.CUDA: [CutlassScaledMMLinearKernel],
18
+ # TODO(rob): Create TritonScaledMMLinear kernel. ROCM will
19
+ # incorrectly attempt to run AZP models if prompted to.
20
+ PlatformEnum.ROCM: [CutlassScaledMMLinearKernel],
21
+ PlatformEnum.TPU: [XLAScaledMMLinearKernel],
22
+}
23
+
24
+
25
+def choose_scaled_mm_linear_kernel(
26
+ config: ScaledMMLinearLayerConfig,
27
+ compute_capability: Optional[int] = None
28
+) -> Type[ScaledMMLinearKernel]:
29
+ """
30
+ Choose an ScalledMMLinearKernel that can implement the given config for the
31
+ given compute capability. Attempts to choose the best kernel in terms of
32
+ performance.
33
+
34
+ Args:
35
+ config (ScaledMMLinearLayerConfig): Description of the linear layer
36
+ to be implemented.
37
+ compute_capability (Optional[int], optional): The compute capability of
38
+ the target device, if None uses `current_platform` to get the
39
+ compute capability. Defaults to None.
40
+
41
+ Raises:
42
+ ValueError: If no kernel can implement the given config.
43
+
44
+ Returns:
45
+ Type[ScaledMMLinearKernel]: Chosen kernel.
46
+ """
47
+
48
+ if compute_capability is None:
49
+ _cc = current_platform.get_device_capability()
50
+ if _cc is not None:
51
+ compute_capability = _cc[0] * 10 + _cc[1]
52
+
53
+ failure_reasons = []
54
+ for kernel in _POSSIBLE_KERNELS[current_platform._enum]:
55
+ if kernel.__name__ in os.environ.get("VLLM_DISABLED_KERNELS", "")\
56
+ .split(","):
57
+ failure_reasons.append(
58
+ f' {kernel.__name__} disabled by environment variable')
59
+ continue
60
+
61
+ # If the current platform uses compute_capability,
62
+ # make sure the kernel supports the compute cability.
63
+ if compute_capability is not None:
64
+ kernel_min_capability = kernel.get_min_capability()
65
+ if (kernel_min_capability is not None
66
+ and kernel_min_capability > compute_capability):
67
+ failure_reasons.append(
68
+ f"{kernel.__name__} requires capability "
69
+ f"{kernel_min_capability}, current compute capability "
70
+ f"is {compute_capability}")
71
+ continue
72
+
73
+ can_implement, failure_reason = kernel.can_implement(config)
74
+ if can_implement:
75
+ return kernel
76
+ else:
77
+ failure_reasons.append(
78
+ f' {kernel.__name__} cannot implement due to: {failure_reason}'
79
+ )
80
+
81
+ raise ValueError(
82
+ "Failed to find a kernel that can implement the "\
83
+ "ScaledMM linear layer. Reasons: \n"
84
+ + '\n'.join(failure_reasons))
vllm/model_executor/layers/quantization/kernels/scaled_mm/cutlass.py
new
+134
@@ -0,0 +1,134 @@
1
+from typing import Optional, Tuple
2
+
3
+import torch
4
+
5
+from vllm import _custom_ops as ops
6
+from vllm.model_executor.layers.quantization.utils import replace_parameter
7
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
8
+ convert_to_channelwise)
9
+from vllm.platforms import current_platform
10
+
11
+from .ScaledMMLinearKernel import (ScaledMMLinearKernel,
12
+ ScaledMMLinearLayerConfig)
13
+
14
+
15
+class CutlassScaledMMLinearKernel(ScaledMMLinearKernel):
16
+
17
+ @classmethod
18
+ def get_min_capability(cls) -> int:
19
+ return 75
20
+
21
+ @classmethod
22
+ def can_implement(
23
+ cls, c: ScaledMMLinearLayerConfig) -> Tuple[bool, Optional[str]]:
24
+
25
+ if (not current_platform.is_cuda() and not current_platform.is_cpu()):
26
+ return False, "CutlassScaledMM requires running on CUDA or CPU."
27
+
28
+ return True, None
29
+
30
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
31
+ # WEIGHT
32
+ # Cutlass kernels need transposed weight.
33
+ weight = getattr(layer, self.w_q_name)
34
+ replace_parameter(
35
+ layer, self.w_q_name,
36
+ torch.nn.Parameter(weight.t().data, requires_grad=False))
37
+
38
+ # WEIGHT SCALE
39
+ # Cutlass kernels support only per-tensor and per-channel.
40
+ # If we have a fused module (QKV, MLP) with per tensor scales (thus N
41
+ # scales being passed to the kernel), convert to the per-channel case.
42
+ is_fused_module = len(layer.logical_widths) > 1
43
+ weight_scale = getattr(layer, self.w_s_name)
44
+ if is_fused_module and not self.config.is_channelwise:
45
+ weight_scale = convert_to_channelwise(weight_scale,
46
+ layer.logical_widths)
47
+ replace_parameter(
48
+ layer, self.w_s_name,
49
+ torch.nn.Parameter(weight_scale.data, requires_grad=False))
50
+
51
+ # INPUT SCALE
52
+ if self.config.is_static_input_scheme:
53
+ input_scale = getattr(layer, self.i_s_name)
54
+
55
+ if self.config.input_symmetric:
56
+ replace_parameter(
57
+ layer, self.i_s_name,
58
+ torch.nn.Parameter(input_scale.max(), requires_grad=False))
59
+ setattr(layer, self.i_zp_name, None)
60
+ else:
61
+ input_zero_point = getattr(layer, self.i_zp_name)
62
+
63
+ # reconstruct the ranges
64
+ int8_traits = torch.iinfo(torch.int8)
65
+ azps = input_zero_point.to(dtype=torch.int32)
66
+ range_max = (input_scale * (int8_traits.max - azps)).max()
67
+ range_min = (input_scale * (int8_traits.min - azps)).min()
68
+
69
+ scale = (range_max - range_min) / (int8_traits.max -
70
+ int8_traits.min)
71
+ replace_parameter(
72
+ layer, self.i_s_name,
73
+ torch.nn.Parameter(scale, requires_grad=False))
74
+
75
+ # AZP loaded as int8 but used as int32
76
+ azp = (int8_traits.min -
77
+ range_min / scale).to(dtype=torch.int32)
78
+ replace_parameter(layer, self.i_zp_name,
79
+ torch.nn.Parameter(azp, requires_grad=False))
80
+
81
+ else:
82
+ setattr(layer, self.i_s_name, None)
83
+ setattr(layer, self.i_zp_name, None)
84
+
85
+ # azp_adj is the AZP adjustment term, used to account for weights.
86
+ # It does not depend on scales or azp, so it is the same for
87
+ # static and dynamic quantization.
88
+ # For more details, see csrc/quantization/cutlass_w8a8/Epilogues.md
89
+ # https://github.com/vllm-project/vllm/blob/8d59dbb00044a588cab96bcdc028006ed922eb06/csrc/quantization/cutlass_w8a8/Epilogues.md
90
+ if not self.config.input_symmetric:
91
+ weight = getattr(layer, self.w_q_name)
92
+ azp_adj = weight.sum(dim=0, keepdim=True, dtype=torch.int32)
93
+ if self.config.is_static_input_scheme:
94
+ # cutlass_w8a8 requires azp to be folded into azp_adj
95
+ # in the per-tensor case
96
+ azp_adj = getattr(layer, self.i_zp_name) * azp_adj
97
+ setattr(layer, self.azp_adj_name,
98
+ torch.nn.Parameter(azp_adj, requires_grad=False))
99
+ else:
100
+ setattr(layer, self.azp_adj_name, None)
101
+
102
+ def apply_weights(self,
103
+ layer: torch.nn.Module,
104
+ x: torch.Tensor,
105
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
106
+ w_q, w_s, i_s, i_zp, azp_adj = self._get_weight_params(layer)
107
+
108
+ # ops.scaled_int8_quant supports both dynamic and static quant:
109
+ # * dynamic, i_s is None and x_s computed from x.
110
+ # * static, i_s is scalar and x_s is i_s.
111
+ symmetric = azp_adj is None
112
+ x_q, x_s, x_zp = ops.scaled_int8_quant(x,
113
+ i_s,
114
+ i_zp,
115
+ symmetric=symmetric)
116
+
117
+ if x_zp is not None:
118
+ # Currently, static is always per-tensor and dynamic is per-token
119
+ static = i_zp is not None
120
+ azp = None if static else x_zp
121
+ return ops.cutlass_scaled_mm_azp(x_q,
122
+ w_q,
123
+ scale_a=x_s,
124
+ scale_b=w_s,
125
+ out_dtype=x.dtype,
126
+ azp_adj=azp_adj,
127
+ azp=azp,
128
+ bias=bias)
129
+ return ops.cutlass_scaled_mm(x_q,
130
+ w_q,
131
+ scale_a=x_s,
132
+ scale_b=w_s,
133
+ out_dtype=x.dtype,
134
+ bias=bias)
vllm/model_executor/layers/quantization/kernels/scaled_mm/xla.py
new
+101
@@ -0,0 +1,101 @@
1
+import warnings
2
+from typing import Optional, Tuple
3
+
4
+import torch
5
+from functorch.experimental.control_flow import cond # noqa: F401
6
+
7
+from vllm.model_executor.layers.quantization.utils import replace_parameter
8
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
9
+ convert_to_channelwise)
10
+from vllm.platforms import current_platform
11
+
12
+from .ScaledMMLinearKernel import (ScaledMMLinearKernel,
13
+ ScaledMMLinearLayerConfig)
14
+
15
+
16
+class XLAScaledMMLinearKernel(ScaledMMLinearKernel):
17
+
18
+ @classmethod
19
+ def get_min_capability(cls) -> int:
20
+ raise NotImplementedError(
21
+ "TPU platform does have a concept of compute capability, "
22
+ "this method should not be called.")
23
+
24
+ @classmethod
25
+ def can_implement(
26
+ cls, c: ScaledMMLinearLayerConfig) -> Tuple[bool, Optional[str]]:
27
+
28
+ if not current_platform.is_tpu():
29
+ return False, "ScaledMMXLA requires running on TPU."
30
+
31
+ if c.is_static_input_scheme:
32
+ return False, "ScaledMMXLA requires dynamic activation scales."
33
+
34
+ if not c.input_symmetric:
35
+ return False, "ScaledMMXLA requires symmetric activation scales."
36
+
37
+ if not c.is_channelwise:
38
+ return False, "ScaledMMXLA requires channelwise weight scales"
39
+
40
+ return True, None
41
+
42
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
43
+ # WEIGHT
44
+ # [out, in] (different than cutlass_scaled_mm)
45
+ weight = getattr(layer, self.w_q_name)
46
+ replace_parameter(layer, self.w_q_name,
47
+ torch.nn.Parameter(weight.data, requires_grad=False))
48
+
49
+ # WEIGHT SCALE
50
+ # XLA kernels support only per-tensor and per-channel.
51
+ # If we have a fused module (QKV, MLP) with per tensor scales (thus N
52
+ # scales being passed to the kernel), convert to the per-channel case.
53
+ is_fused_module = len(layer.logical_widths) > 1
54
+ weight_scale = getattr(layer, self.w_s_name)
55
+ if is_fused_module and not self.config.is_channelwise:
56
+ weight_scale = convert_to_channelwise(weight_scale,
57
+ layer.logical_widths)
58
+
59
+ # [out_channel,] (different than cutlass_scaled_mm)
60
+ weight_scale = weight_scale.squeeze(-1)
61
+ replace_parameter(
62
+ layer, self.w_s_name,
63
+ torch.nn.Parameter(weight_scale.data, requires_grad=False))
64
+
65
+ # Only support symmetric dynamic activation quantization.
66
+ setattr(layer, self.i_s_name, None)
67
+ setattr(layer, self.i_zp_name, None)
68
+ setattr(layer, self.azp_adj_name, None)
69
+
70
+ # Filter warning for cond usage in apply_weights. It is okay
71
+ # to specialize the graph since bias is not dynamic.
72
+ warnings.filterwarnings(
73
+ "ignore",
74
+ message=
75
+ "Pred is a Python constant. When used with torch.cond, it specializes on one of the branches." # noqa: E501
76
+ )
77
+
78
+ def no_add_bias(self, x: torch.Tensor, bias: Optional[torch.Tensor]):
79
+ return x
80
+
81
+ def add_bias(self, x: torch.Tensor, bias: Optional[torch.Tensor]):
82
+ return x + bias
83
+
84
+ def apply_weights(self,
85
+ layer: torch.nn.Module,
86
+ x: torch.Tensor,
87
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
88
+ w_q, w_s, _, _, _ = self._get_weight_params(layer)
89
+
90
+ import torch_xla.experimental.xla_quantized_matmul # noqa: F401
91
+ out = torch.ops.xla.quantized_matmul(x,
92
+ w_q,
93
+ w_s,
94
+ zero_point=None,
95
+ block_size=-1,
96
+ int4_weight=False,
97
+ quantize_activation=True)
98
+
99
+ # Explicitly capture control flow to make dynamo happy.
100
+ # https://pytorch.org/docs/main/generated/exportdb/index.html#cond-branch-class-method # noqa: E501
101
+ return cond(bias is None, self.no_add_bias, self.add_bias, [out, bias])
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
-38
@@ -201,44 +201,6 @@ def apply_fp8_linear(
201
return output.to(dtype=input.dtype).view(*output_shape)
202
203
204
-def apply_int8_linear(
205
- input: torch.Tensor,
206
- weight: torch.Tensor,
207
- weight_scale: torch.Tensor,
208
- input_scale: Optional[torch.Tensor] = None,
209
- input_zero_point: Optional[torch.Tensor] = None,
210
- azp_adj: Optional[torch.Tensor] = None,
211
- bias: Optional[torch.Tensor] = None,
212
-):
213
- # ops.scaled_int8_quant supports both dynamic and static quant.
214
- # * dynamic, layer.input_scale is None and x_scale computed from x.
215
- # * static, layer.input_scale is scalar and x_scale is input_scale.
216
- symmetric = azp_adj is None
217
- x_q, x_scale, x_zp = ops.scaled_int8_quant(input,
218
- input_scale,
219
- input_zero_point,
220
- symmetric=symmetric)
221
-
222
- if x_zp is not None:
223
- # Currently, static is always per-tensor and dynamic is per-token
224
- static = input_zero_point is not None
225
- azp = None if static else x_zp
226
- return ops.cutlass_scaled_mm_azp(x_q,
227
- weight,
228
- scale_a=x_scale,
229
- scale_b=weight_scale,
230
- out_dtype=input.dtype,
231
- azp_adj=azp_adj,
232
- azp=azp,
233
- bias=bias)
234
- return ops.cutlass_scaled_mm(x_q,
235
- weight,
236
- scale_a=x_scale,
237
- scale_b=weight_scale,
238
- out_dtype=input.dtype,
239
- bias=bias)
240
-
241
-
204
def normalize_e4m3fn_to_e4m3fnuz(
205
weight: torch.Tensor,
206
weight_scale: torch.Tensor,
vllm/model_executor/parameter.py
+13
@@ -6,6 +6,7 @@ from torch.nn import Parameter
6
7
from vllm.distributed import get_tensor_model_parallel_rank
8
from vllm.logger import init_logger
9
+from vllm.model_executor.utils import _make_synced_weight_loader
10
11
__all__ = [
12
"BasevLLMParameter", "PackedvLLMParameter", "PerTensorScaleParameter",
@@ -37,6 +38,18 @@ class BasevLLMParameter(Parameter):
38
:returns: a torch.nn.parameter
39
"""
40
41
+ # During weight loading, we often do something like:
42
+ # narrowed_tensor = param.data.narrow(0, offset, len)
43
+ # narrowed_tensor.copy_(real_weight)
44
+ # expecting narrowed_tensor and param.data to share the same storage.
45
+ # However, on TPUs, narrowed_tensor will lazily propagate to the base
46
+ # tensor, which is param.data, leading to the redundant memory usage.
47
+ # This sometimes causes OOM errors during model loading. To avoid this,
48
+ # we sync the param tensor after its weight loader is called.
49
+ from vllm.platforms import current_platform
50
+ if current_platform.is_tpu():
51
+ weight_loader = _make_synced_weight_loader(weight_loader)
52
+
53
self._weight_loader = weight_loader
54
55
@property
vllm/platforms/tpu.py
+3
-1
@@ -19,7 +19,9 @@ class TpuPlatform(Platform):
19
device_name: str = "tpu"
20
device_type: str = "tpu"
21
dispatch_key: str = "XLA"
22
- supported_quantization: list[str] = ["tpu_int8"]
22
+ supported_quantization: list[str] = [
23
+ "tpu_int8", "compressed-tensors", "compressed_tensors"
24
+ ]
25
26
@classmethod
27
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend: