@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 56fe4c297c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[TPU][Quantization] TPU `W8A8` (#11785)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Robert Shaw committed Jan 8, 2025 at 14:33 UTC 56fe4c297c7d9d872eccc19e3edbf1d75e1a30e2
18 files changed +565 -190
.buildkite/run-tpu-test.sh
+10 -1
@@ -14,4 +14,13 @@ remove_docker_container
14 # For HF_TOKEN.
15 source /etc/environment
16 # Run a simple end-to-end example.
17 -docker run --privileged --net host --shm-size=16G -it -e "HF_TOKEN=$HF_TOKEN" --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && python3 -m pip install lm_eval[api]==0.4.4 && pytest -v -s /workspace/vllm/tests/entrypoints/openai/test_accuracy.py && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference/offline_inference_tpu.py"
17 +docker run --privileged --net host --shm-size=16G -it \
18 + -e "HF_TOKEN=$HF_TOKEN" --name tpu-test \
19 + vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git \
20 + && python3 -m pip install pytest \
21 + && python3 -m pip install lm_eval[api]==0.4.4 \
22 + && pytest -v -s /workspace/vllm/tests/entrypoints/openai/test_accuracy.py \
23 + && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py \
24 + && python3 /workspace/vllm/tests/tpu/test_compilation.py \
25 + && python3 /workspace/vllm/tests/tpu/test_quantization_accuracy.py \
26 + && python3 /workspace/vllm/examples/offline_inference/offline_inference_tpu.py"
tests/tpu/test_quantization_accuracy.py new
+49
@@ -0,0 +1,49 @@
1 +from dataclasses import dataclass
2 +
3 +import lm_eval
4 +import pytest
5 +
6 +TASK = "gsm8k"
7 +FILTER = "exact_match,strict-match"
8 +RTOL = 0.03
9 +
10 +
11 +@dataclass
12 +class GSM8KAccuracyTestConfig:
13 + model_name: str
14 + excepted_value: float
15 +
16 + def get_model_args(self) -> str:
17 + return (f"pretrained={self.model_name},"
18 + "max_model_len=4096,max_num_seqs=32")
19 +
20 +
21 +# NOTE: Accuracy scores measured on GPUs.
22 +ACCURACY_CONFIGS = [
23 + GSM8KAccuracyTestConfig(
24 + model_name="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
25 + excepted_value=0.76), # no bias
26 + # NOTE(rob): We cannot re-initialize VLLM in the same process for TPU,
27 + # so only one of these tests can run in a single call to pytest. As
28 + # a follow up, move this into the LM-EVAL section of the CI.
29 + # GSM8KAccuracyTestConfig(
30 + # model_name="neuralmagic/Qwen2-7B-Instruct-quantized.w8a8",
31 + # excepted_value=0.66), # bias in QKV layers
32 +]
33 +
34 +
35 +@pytest.mark.parametrize("config", ACCURACY_CONFIGS)
36 +def test_gsm8k_correctness(config: GSM8KAccuracyTestConfig):
37 +
38 + results = lm_eval.simple_evaluate(
39 + model="vllm",
40 + model_args=config.get_model_args(),
41 + tasks="gsm8k",
42 + batch_size="auto",
43 + )
44 +
45 + EXPECTED_VALUE = config.excepted_value
46 + measured_value = results["results"][TASK][FILTER]
47 + assert (measured_value - RTOL < EXPECTED_VALUE
48 + and measured_value + RTOL > EXPECTED_VALUE
49 + ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+31 -74
@@ -1,14 +1,13 @@
1 -from typing import Callable, List, Optional
1 +from typing import Callable, List, Optional, Set
2
3 import torch
4 from compressed_tensors.quantization import QuantizationStrategy
5 -from torch.nn import Parameter
5
6 from vllm.logger import init_logger
7 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 CompressedTensorsScheme)
10 -from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
11 - apply_int8_linear, convert_to_channelwise)
9 +from vllm.model_executor.layers.quantization.kernels.scaled_mm import (
10 + ScaledMMLinearLayerConfig, choose_scaled_mm_linear_kernel)
11 from vllm.model_executor.parameter import (BasevLLMParameter,
12 ChannelQuantScaleParameter,
13 ModelWeightParameter,
@@ -18,6 +17,7 @@ logger = init_logger(__name__)
17
18
19 class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
20 + _kernel_backends_being_used: Set[str] = set()
21
22 def __init__(self, strategy: str, is_static_input_scheme: bool,
23 input_symmetric: bool):
@@ -30,74 +30,25 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
30 # turing and up
31 return 75
32
33 - def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
34 - # WEIGHT
35 - # Cutlass kernels need transposed weight.
36 - weight = layer.weight
37 - layer.weight = Parameter(weight.t(), requires_grad=False)
38 -
39 - # WEIGHT SCALE
40 - # Cutlass kernels support only per-tensor and per-channel.
41 - # If we have a fused module (QKV, MLP) with per tensor scales (thus N
42 - # scales being passed to the kernel), convert to the per-channel case.
43 - is_fused_module = len(self.logical_widths) > 1
44 - if is_fused_module and self.strategy == QuantizationStrategy.TENSOR:
45 - ws_channelwise = convert_to_channelwise(layer.weight_scale,
46 - self.logical_widths)
47 - layer.weight_scale = Parameter(ws_channelwise, requires_grad=False)
48 - else:
49 - layer.weight_scale = Parameter(layer.weight_scale.data,
50 - requires_grad=False)
51 - # INPUT SCALE
52 - if self.is_static_input_scheme:
53 - if self.input_symmetric:
54 - layer.input_scale = Parameter(layer.input_scale.max(),
55 - requires_grad=False)
56 - layer.input_zero_point = None
57 - else:
58 - # reconstruct the ranges
59 - int8_traits = torch.iinfo(torch.int8)
60 - azps = layer.input_zero_point.to(dtype=torch.int32)
61 - range_max = (layer.input_scale *
62 - (int8_traits.max - azps)).max()
63 - range_min = (layer.input_scale *
64 - (int8_traits.min - azps)).min()
65 -
66 - scale = (range_max - range_min) / (int8_traits.max -
67 - int8_traits.min)
68 - layer.input_scale = Parameter(scale, requires_grad=False)
69 -
70 - # AZP loaded as int8 but used as int32
71 - azp = (int8_traits.min -
72 - range_min / scale).to(dtype=torch.int32)
73 - layer.input_zero_point = Parameter(azp, requires_grad=False)
74 -
75 - else:
76 - layer.input_scale = None
77 - layer.input_zero_point = None
78 -
79 - # azp_adj is the AZP adjustment term, used to account for weights.
80 - # It does not depend on scales or azp, so it is the same for
81 - # static and dynamic quantization.
82 - # For more details, see csrc/quantization/cutlass_w8a8/Epilogues.md
83 - # https://github.com/vllm-project/vllm/blob/8d59dbb00044a588cab96bcdc028006ed922eb06/csrc/quantization/cutlass_w8a8/Epilogues.md
84 - if not self.input_symmetric:
85 - azp_adj = layer.weight.sum(dim=0, keepdim=True, dtype=torch.int32)
86 - if self.is_static_input_scheme:
87 - # cutlass_w8a8 requires azp to be folded into azp_adj
88 - # in the per-tensor case
89 - azp_adj = layer.input_zero_point * azp_adj
90 -
91 - layer.azp_adj = azp_adj
92 - else:
93 - layer.azp_adj = None
94 -
33 def create_weights(self, layer: torch.nn.Module,
34 output_partition_sizes: List[int],
35 input_size_per_partition: int,
36 params_dtype: torch.dtype, weight_loader: Callable,
37 **kwargs):
100 - self.logical_widths = output_partition_sizes
38 + layer.logical_widths = output_partition_sizes
39 +
40 + scaled_mm_linear_kernel_config = ScaledMMLinearLayerConfig(
41 + is_channelwise=(self.strategy == QuantizationStrategy.CHANNEL),
42 + is_static_input_scheme=self.is_static_input_scheme,
43 + input_symmetric=self.input_symmetric)
44 +
45 + kernel_type = choose_scaled_mm_linear_kernel(
46 + scaled_mm_linear_kernel_config)
47 +
48 + if kernel_type.__name__ not in self._kernel_backends_being_used:
49 + logger.info("Using %s for CompressedTensorsW8A8Int8",
50 + kernel_type.__name__)
51 + self._kernel_backends_being_used.add(kernel_type.__name__)
52
53 # WEIGHT
54 weight = ModelWeightParameter(data=torch.empty(
@@ -140,12 +91,18 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
91 weight_loader=weight_loader)
92 layer.register_parameter("input_zero_point", input_zero_point)
93
94 + self.kernel = kernel_type(c=scaled_mm_linear_kernel_config,
95 + w_q_param_name="weight",
96 + w_s_param_name="weight_scale",
97 + i_s_param_name="input_scale",
98 + i_zp_param_name="input_zero_point",
99 + azp_adj_param_name="azp_adj")
100 +
101 + # Checkpoints are serialized in compressed-tensors format, which is
102 + # different from the format the kernel may want. Handle repacking here.
103 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
104 + self.kernel.process_weights_after_loading(layer)
105 +
106 def apply_weights(self, layer: torch.nn.Module, x: torch.Tensor,
107 bias: Optional[torch.Tensor]) -> torch.Tensor:
145 - return apply_int8_linear(input=x,
146 - weight=layer.weight,
147 - weight_scale=layer.weight_scale,
148 - input_scale=layer.input_scale,
149 - input_zero_point=layer.input_zero_point,
150 - azp_adj=layer.azp_adj,
151 - bias=bias)
108 + return self.kernel.apply_weights(layer, x, bias)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+1 -1
@@ -6,7 +6,7 @@ from compressed_tensors.quantization import ActivationOrdering
6 from vllm.logger import init_logger
7 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
8 CompressedTensorsScheme)
9 -from vllm.model_executor.layers.quantization.kernels import (
9 +from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
10 MPLinearLayerConfig, choose_mp_linear_kernel)
11 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12 marlin_repeat_scales_on_all_ranks)
vllm/model_executor/layers/quantization/gptq_marlin.py
+1 -1
@@ -11,7 +11,7 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
11 set_weight_attrs)
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig)
14 -from vllm.model_executor.layers.quantization.kernels import (
14 +from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
15 MPLinearLayerConfig, choose_mp_linear_kernel)
16 from vllm.model_executor.layers.quantization.utils import replace_parameter
17 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
vllm/model_executor/layers/quantization/kernels/__init__.py
-74
@@ -1,74 +0,0 @@
1 -from typing import List, Optional, Type
2 -
3 -import vllm.envs as envs
4 -from vllm.model_executor.layers.quantization.kernels.exllama import (
5 - ExllamaLinearKernel)
6 -from vllm.model_executor.layers.quantization.kernels.machete import (
7 - MacheteLinearKernel)
8 -from vllm.model_executor.layers.quantization.kernels.marlin import (
9 - MarlinLinearKernel)
10 -from vllm.model_executor.layers.quantization.kernels.MPLinearKernel import (
11 - MPLinearKernel, MPLinearLayerConfig)
12 -from vllm.platforms import current_platform
13 -
14 -# in priority/performance order (when available)
15 -_POSSIBLE_KERNELS: List[Type[MPLinearKernel]] = [
16 - MacheteLinearKernel,
17 - MarlinLinearKernel,
18 - ExllamaLinearKernel,
19 -]
20 -
21 -
22 -def choose_mp_linear_kernel(
23 - config: MPLinearLayerConfig,
24 - compute_capability: Optional[int] = None) -> Type[MPLinearKernel]:
25 - """
26 - Choose an MPLinearKernel that can implement the given config for the given
27 - compute capability. Attempts to choose the best kernel in terms of
28 - performance.
29 -
30 - Args:
31 - config (MPLinearLayerConfig): Description of the linear layer to be
32 - implemented.
33 - compute_capability (Optional[int], optional): The compute capability of
34 - the target device, if None uses `current_platform` to get the compute
35 - capability. Defaults to None.
36 -
37 - Raises:
38 - ValueError: If no kernel can implement the given config.
39 -
40 - Returns:
41 - Type[MPLinearKernel]: Chosen kernel.
42 - """
43 - if compute_capability is None:
44 - if current_platform is None:
45 - raise ValueError("Cannot determine compute capability")
46 - _cc = current_platform.get_device_capability()
47 - compute_capability = _cc[0] * 10 + _cc[1]
48 -
49 - failure_reasons = []
50 - for kernel in _POSSIBLE_KERNELS:
51 - if kernel.__name__ in envs.VLLM_DISABLED_KERNELS:
52 - failure_reasons.append(
53 - f' {kernel.__name__} disabled by environment variable')
54 - continue
55 -
56 - if kernel.get_min_capability() > compute_capability:
57 - failure_reasons.append(
58 - f"{kernel.__name__} requires capability "
59 - f"{kernel.get_min_capability()}, current compute capability "
60 - f"is {compute_capability}")
61 - continue
62 -
63 - can_implement, failure_reason = kernel.can_implement(config)
64 - if can_implement:
65 - return kernel
66 - else:
67 - failure_reasons.append(
68 - f' {kernel.__name__} cannot implement due to: {failure_reason}'
69 - )
70 -
71 - raise ValueError(
72 - "Failed to find a kernel that can implement the "\
73 - "WNA16 linear layer. Reasons: \n"
74 - + '\n'.join(failure_reasons))
vllm/model_executor/layers/quantization/kernels/mixed_precision/MPLinearKernel.py renamed
vllm/model_executor/layers/quantization/kernels/mixed_precision/__init__.py new
+74
@@ -0,0 +1,74 @@
1 +from typing import List, Optional, Type
2 +
3 +import vllm.envs as envs
4 +from vllm.model_executor.layers.quantization.kernels.mixed_precision.exllama import ( # noqa: E501
5 + ExllamaLinearKernel)
6 +from vllm.model_executor.layers.quantization.kernels.mixed_precision.machete import ( # noqa: E501
7 + MacheteLinearKernel)
8 +from vllm.model_executor.layers.quantization.kernels.mixed_precision.marlin import ( # noqa: E501
9 + MarlinLinearKernel)
10 +from vllm.model_executor.layers.quantization.kernels.mixed_precision.MPLinearKernel import ( # noqa: E501
11 + MPLinearKernel, MPLinearLayerConfig)
12 +from vllm.platforms import current_platform
13 +
14 +# in priority/performance order (when available)
15 +_POSSIBLE_KERNELS: List[Type[MPLinearKernel]] = [
16 + MacheteLinearKernel,
17 + MarlinLinearKernel,
18 + ExllamaLinearKernel,
19 +]
20 +
21 +
22 +def choose_mp_linear_kernel(
23 + config: MPLinearLayerConfig,
24 + compute_capability: Optional[int] = None) -> Type[MPLinearKernel]:
25 + """
26 + Choose an MPLinearKernel that can implement the given config for the given
27 + compute capability. Attempts to choose the best kernel in terms of
28 + performance.
29 +
30 + Args:
31 + config (MPLinearLayerConfig): Description of the linear layer to be
32 + implemented.
33 + compute_capability (Optional[int], optional): The compute capability of
34 + the target device, if None uses `current_platform` to get the compute
35 + capability. Defaults to None.
36 +
37 + Raises:
38 + ValueError: If no kernel can implement the given config.
39 +
40 + Returns:
41 + Type[MPLinearKernel]: Chosen kernel.
42 + """
43 + if compute_capability is None:
44 + if current_platform is None:
45 + raise ValueError("Cannot determine compute capability")
46 + _cc = current_platform.get_device_capability()
47 + compute_capability = _cc[0] * 10 + _cc[1]
48 +
49 + failure_reasons = []
50 + for kernel in _POSSIBLE_KERNELS:
51 + if kernel.__name__ in envs.VLLM_DISABLED_KERNELS:
52 + failure_reasons.append(
53 + f' {kernel.__name__} disabled by environment variable')
54 + continue
55 +
56 + if kernel.get_min_capability() > compute_capability:
57 + failure_reasons.append(
58 + f"{kernel.__name__} requires capability "
59 + f"{kernel.get_min_capability()}, current compute capability "
60 + f"is {compute_capability}")
61 + continue
62 +
63 + can_implement, failure_reason = kernel.can_implement(config)
64 + if can_implement:
65 + return kernel
66 + else:
67 + failure_reasons.append(
68 + f' {kernel.__name__} cannot implement due to: {failure_reason}'
69 + )
70 +
71 + raise ValueError(
72 + "Failed to find a kernel that can implement the "\
73 + "WNA16 linear layer. Reasons: \n"
74 + + '\n'.join(failure_reasons))
vllm/model_executor/layers/quantization/kernels/mixed_precision/exllama.py renamed
vllm/model_executor/layers/quantization/kernels/mixed_precision/machete.py renamed
vllm/model_executor/layers/quantization/kernels/mixed_precision/marlin.py renamed
vllm/model_executor/layers/quantization/kernels/scaled_mm/ScaledMMLinearKernel.py new
+64
@@ -0,0 +1,64 @@
1 +from abc import ABC, abstractmethod
2 +from dataclasses import dataclass
3 +from typing import Optional, Tuple
4 +
5 +import torch
6 +
7 +
8 +@dataclass
9 +class ScaledMMLinearLayerConfig:
10 + is_channelwise: bool
11 + is_static_input_scheme: bool
12 + input_symmetric: bool
13 +
14 +
15 +class ScaledMMLinearKernel(ABC):
16 +
17 + @classmethod
18 + @abstractmethod
19 + def get_min_capability(cls) -> int:
20 + raise NotImplementedError
21 +
22 + @classmethod
23 + @abstractmethod
24 + def can_implement(
25 + cls, c: ScaledMMLinearLayerConfig) -> Tuple[bool, Optional[str]]:
26 + raise NotImplementedError
27 +
28 + def __init__(self, c: ScaledMMLinearLayerConfig, w_q_param_name: str,
29 + w_s_param_name: str, i_s_param_name: str,
30 + i_zp_param_name: str, azp_adj_param_name: str) -> None:
31 + assert self.can_implement(c)
32 + self.config = c
33 + self.w_q_name = w_q_param_name
34 + self.w_s_name = w_s_param_name
35 + self.i_s_name = i_s_param_name
36 + self.i_zp_name = i_zp_param_name
37 + self.azp_adj_name = azp_adj_param_name
38 +
39 + @abstractmethod
40 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
41 + raise NotImplementedError
42 +
43 + @abstractmethod
44 + def apply_weights(self,
45 + layer: torch.nn.Module,
46 + x: torch.Tensor,
47 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
48 + raise NotImplementedError
49 +
50 + def _get_weight_params(
51 + self, layer: torch.nn.Module
52 + ) -> Tuple[torch.Tensor, # weight
53 + torch.Tensor, # weight_scale
54 + Optional[torch.Tensor], # input_scale,
55 + Optional[torch.Tensor], # input_zp
56 + Optional[torch.Tensor], # azp_adj
57 + ]:
58 + return (
59 + getattr(layer, self.w_q_name),
60 + getattr(layer, self.w_s_name),
61 + getattr(layer, self.i_s_name),
62 + getattr(layer, self.i_zp_name),
63 + getattr(layer, self.azp_adj_name),
64 + )
vllm/model_executor/layers/quantization/kernels/scaled_mm/__init__.py new
+84
@@ -0,0 +1,84 @@
1 +import os
2 +from typing import Dict, List, Optional, Type
3 +
4 +from vllm.model_executor.layers.quantization.kernels.scaled_mm.cutlass import (
5 + CutlassScaledMMLinearKernel)
6 +from vllm.model_executor.layers.quantization.kernels.scaled_mm.ScaledMMLinearKernel import ( # noqa: E501
7 + ScaledMMLinearKernel, ScaledMMLinearLayerConfig)
8 +# from vllm.model_executor.layers.quantization.kernels.scaled_mm.triton import (
9 +# TritonScaledMMLinear)
10 +from vllm.model_executor.layers.quantization.kernels.scaled_mm.xla import (
11 + XLAScaledMMLinearKernel)
12 +from vllm.platforms import PlatformEnum, current_platform
13 +
14 +# in priority/performance order (when available)
15 +_POSSIBLE_KERNELS: Dict[PlatformEnum, List[Type[ScaledMMLinearKernel]]] = {
16 + PlatformEnum.CPU: [CutlassScaledMMLinearKernel],
17 + PlatformEnum.CUDA: [CutlassScaledMMLinearKernel],
18 + # TODO(rob): Create TritonScaledMMLinear kernel. ROCM will
19 + # incorrectly attempt to run AZP models if prompted to.
20 + PlatformEnum.ROCM: [CutlassScaledMMLinearKernel],
21 + PlatformEnum.TPU: [XLAScaledMMLinearKernel],
22 +}
23 +
24 +
25 +def choose_scaled_mm_linear_kernel(
26 + config: ScaledMMLinearLayerConfig,
27 + compute_capability: Optional[int] = None
28 +) -> Type[ScaledMMLinearKernel]:
29 + """
30 + Choose an ScalledMMLinearKernel that can implement the given config for the
31 + given compute capability. Attempts to choose the best kernel in terms of
32 + performance.
33 +
34 + Args:
35 + config (ScaledMMLinearLayerConfig): Description of the linear layer
36 + to be implemented.
37 + compute_capability (Optional[int], optional): The compute capability of
38 + the target device, if None uses `current_platform` to get the
39 + compute capability. Defaults to None.
40 +
41 + Raises:
42 + ValueError: If no kernel can implement the given config.
43 +
44 + Returns:
45 + Type[ScaledMMLinearKernel]: Chosen kernel.
46 + """
47 +
48 + if compute_capability is None:
49 + _cc = current_platform.get_device_capability()
50 + if _cc is not None:
51 + compute_capability = _cc[0] * 10 + _cc[1]
52 +
53 + failure_reasons = []
54 + for kernel in _POSSIBLE_KERNELS[current_platform._enum]:
55 + if kernel.__name__ in os.environ.get("VLLM_DISABLED_KERNELS", "")\
56 + .split(","):
57 + failure_reasons.append(
58 + f' {kernel.__name__} disabled by environment variable')
59 + continue
60 +
61 + # If the current platform uses compute_capability,
62 + # make sure the kernel supports the compute cability.
63 + if compute_capability is not None:
64 + kernel_min_capability = kernel.get_min_capability()
65 + if (kernel_min_capability is not None
66 + and kernel_min_capability > compute_capability):
67 + failure_reasons.append(
68 + f"{kernel.__name__} requires capability "
69 + f"{kernel_min_capability}, current compute capability "
70 + f"is {compute_capability}")
71 + continue
72 +
73 + can_implement, failure_reason = kernel.can_implement(config)
74 + if can_implement:
75 + return kernel
76 + else:
77 + failure_reasons.append(
78 + f' {kernel.__name__} cannot implement due to: {failure_reason}'
79 + )
80 +
81 + raise ValueError(
82 + "Failed to find a kernel that can implement the "\
83 + "ScaledMM linear layer. Reasons: \n"
84 + + '\n'.join(failure_reasons))
vllm/model_executor/layers/quantization/kernels/scaled_mm/cutlass.py new
+134
@@ -0,0 +1,134 @@
1 +from typing import Optional, Tuple
2 +
3 +import torch
4 +
5 +from vllm import _custom_ops as ops
6 +from vllm.model_executor.layers.quantization.utils import replace_parameter
7 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
8 + convert_to_channelwise)
9 +from vllm.platforms import current_platform
10 +
11 +from .ScaledMMLinearKernel import (ScaledMMLinearKernel,
12 + ScaledMMLinearLayerConfig)
13 +
14 +
15 +class CutlassScaledMMLinearKernel(ScaledMMLinearKernel):
16 +
17 + @classmethod
18 + def get_min_capability(cls) -> int:
19 + return 75
20 +
21 + @classmethod
22 + def can_implement(
23 + cls, c: ScaledMMLinearLayerConfig) -> Tuple[bool, Optional[str]]:
24 +
25 + if (not current_platform.is_cuda() and not current_platform.is_cpu()):
26 + return False, "CutlassScaledMM requires running on CUDA or CPU."
27 +
28 + return True, None
29 +
30 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
31 + # WEIGHT
32 + # Cutlass kernels need transposed weight.
33 + weight = getattr(layer, self.w_q_name)
34 + replace_parameter(
35 + layer, self.w_q_name,
36 + torch.nn.Parameter(weight.t().data, requires_grad=False))
37 +
38 + # WEIGHT SCALE
39 + # Cutlass kernels support only per-tensor and per-channel.
40 + # If we have a fused module (QKV, MLP) with per tensor scales (thus N
41 + # scales being passed to the kernel), convert to the per-channel case.
42 + is_fused_module = len(layer.logical_widths) > 1
43 + weight_scale = getattr(layer, self.w_s_name)
44 + if is_fused_module and not self.config.is_channelwise:
45 + weight_scale = convert_to_channelwise(weight_scale,
46 + layer.logical_widths)
47 + replace_parameter(
48 + layer, self.w_s_name,
49 + torch.nn.Parameter(weight_scale.data, requires_grad=False))
50 +
51 + # INPUT SCALE
52 + if self.config.is_static_input_scheme:
53 + input_scale = getattr(layer, self.i_s_name)
54 +
55 + if self.config.input_symmetric:
56 + replace_parameter(
57 + layer, self.i_s_name,
58 + torch.nn.Parameter(input_scale.max(), requires_grad=False))
59 + setattr(layer, self.i_zp_name, None)
60 + else:
61 + input_zero_point = getattr(layer, self.i_zp_name)
62 +
63 + # reconstruct the ranges
64 + int8_traits = torch.iinfo(torch.int8)
65 + azps = input_zero_point.to(dtype=torch.int32)
66 + range_max = (input_scale * (int8_traits.max - azps)).max()
67 + range_min = (input_scale * (int8_traits.min - azps)).min()
68 +
69 + scale = (range_max - range_min) / (int8_traits.max -
70 + int8_traits.min)
71 + replace_parameter(
72 + layer, self.i_s_name,
73 + torch.nn.Parameter(scale, requires_grad=False))
74 +
75 + # AZP loaded as int8 but used as int32
76 + azp = (int8_traits.min -
77 + range_min / scale).to(dtype=torch.int32)
78 + replace_parameter(layer, self.i_zp_name,
79 + torch.nn.Parameter(azp, requires_grad=False))
80 +
81 + else:
82 + setattr(layer, self.i_s_name, None)
83 + setattr(layer, self.i_zp_name, None)
84 +
85 + # azp_adj is the AZP adjustment term, used to account for weights.
86 + # It does not depend on scales or azp, so it is the same for
87 + # static and dynamic quantization.
88 + # For more details, see csrc/quantization/cutlass_w8a8/Epilogues.md
89 + # https://github.com/vllm-project/vllm/blob/8d59dbb00044a588cab96bcdc028006ed922eb06/csrc/quantization/cutlass_w8a8/Epilogues.md
90 + if not self.config.input_symmetric:
91 + weight = getattr(layer, self.w_q_name)
92 + azp_adj = weight.sum(dim=0, keepdim=True, dtype=torch.int32)
93 + if self.config.is_static_input_scheme:
94 + # cutlass_w8a8 requires azp to be folded into azp_adj
95 + # in the per-tensor case
96 + azp_adj = getattr(layer, self.i_zp_name) * azp_adj
97 + setattr(layer, self.azp_adj_name,
98 + torch.nn.Parameter(azp_adj, requires_grad=False))
99 + else:
100 + setattr(layer, self.azp_adj_name, None)
101 +
102 + def apply_weights(self,
103 + layer: torch.nn.Module,
104 + x: torch.Tensor,
105 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
106 + w_q, w_s, i_s, i_zp, azp_adj = self._get_weight_params(layer)
107 +
108 + # ops.scaled_int8_quant supports both dynamic and static quant:
109 + # * dynamic, i_s is None and x_s computed from x.
110 + # * static, i_s is scalar and x_s is i_s.
111 + symmetric = azp_adj is None
112 + x_q, x_s, x_zp = ops.scaled_int8_quant(x,
113 + i_s,
114 + i_zp,
115 + symmetric=symmetric)
116 +
117 + if x_zp is not None:
118 + # Currently, static is always per-tensor and dynamic is per-token
119 + static = i_zp is not None
120 + azp = None if static else x_zp
121 + return ops.cutlass_scaled_mm_azp(x_q,
122 + w_q,
123 + scale_a=x_s,
124 + scale_b=w_s,
125 + out_dtype=x.dtype,
126 + azp_adj=azp_adj,
127 + azp=azp,
128 + bias=bias)
129 + return ops.cutlass_scaled_mm(x_q,
130 + w_q,
131 + scale_a=x_s,
132 + scale_b=w_s,
133 + out_dtype=x.dtype,
134 + bias=bias)
vllm/model_executor/layers/quantization/kernels/scaled_mm/xla.py new
+101
@@ -0,0 +1,101 @@
1 +import warnings
2 +from typing import Optional, Tuple
3 +
4 +import torch
5 +from functorch.experimental.control_flow import cond # noqa: F401
6 +
7 +from vllm.model_executor.layers.quantization.utils import replace_parameter
8 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
9 + convert_to_channelwise)
10 +from vllm.platforms import current_platform
11 +
12 +from .ScaledMMLinearKernel import (ScaledMMLinearKernel,
13 + ScaledMMLinearLayerConfig)
14 +
15 +
16 +class XLAScaledMMLinearKernel(ScaledMMLinearKernel):
17 +
18 + @classmethod
19 + def get_min_capability(cls) -> int:
20 + raise NotImplementedError(
21 + "TPU platform does have a concept of compute capability, "
22 + "this method should not be called.")
23 +
24 + @classmethod
25 + def can_implement(
26 + cls, c: ScaledMMLinearLayerConfig) -> Tuple[bool, Optional[str]]:
27 +
28 + if not current_platform.is_tpu():
29 + return False, "ScaledMMXLA requires running on TPU."
30 +
31 + if c.is_static_input_scheme:
32 + return False, "ScaledMMXLA requires dynamic activation scales."
33 +
34 + if not c.input_symmetric:
35 + return False, "ScaledMMXLA requires symmetric activation scales."
36 +
37 + if not c.is_channelwise:
38 + return False, "ScaledMMXLA requires channelwise weight scales"
39 +
40 + return True, None
41 +
42 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
43 + # WEIGHT
44 + # [out, in] (different than cutlass_scaled_mm)
45 + weight = getattr(layer, self.w_q_name)
46 + replace_parameter(layer, self.w_q_name,
47 + torch.nn.Parameter(weight.data, requires_grad=False))
48 +
49 + # WEIGHT SCALE
50 + # XLA kernels support only per-tensor and per-channel.
51 + # If we have a fused module (QKV, MLP) with per tensor scales (thus N
52 + # scales being passed to the kernel), convert to the per-channel case.
53 + is_fused_module = len(layer.logical_widths) > 1
54 + weight_scale = getattr(layer, self.w_s_name)
55 + if is_fused_module and not self.config.is_channelwise:
56 + weight_scale = convert_to_channelwise(weight_scale,
57 + layer.logical_widths)
58 +
59 + # [out_channel,] (different than cutlass_scaled_mm)
60 + weight_scale = weight_scale.squeeze(-1)
61 + replace_parameter(
62 + layer, self.w_s_name,
63 + torch.nn.Parameter(weight_scale.data, requires_grad=False))
64 +
65 + # Only support symmetric dynamic activation quantization.
66 + setattr(layer, self.i_s_name, None)
67 + setattr(layer, self.i_zp_name, None)
68 + setattr(layer, self.azp_adj_name, None)
69 +
70 + # Filter warning for cond usage in apply_weights. It is okay
71 + # to specialize the graph since bias is not dynamic.
72 + warnings.filterwarnings(
73 + "ignore",
74 + message=
75 + "Pred is a Python constant. When used with torch.cond, it specializes on one of the branches." # noqa: E501
76 + )
77 +
78 + def no_add_bias(self, x: torch.Tensor, bias: Optional[torch.Tensor]):
79 + return x
80 +
81 + def add_bias(self, x: torch.Tensor, bias: Optional[torch.Tensor]):
82 + return x + bias
83 +
84 + def apply_weights(self,
85 + layer: torch.nn.Module,
86 + x: torch.Tensor,
87 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
88 + w_q, w_s, _, _, _ = self._get_weight_params(layer)
89 +
90 + import torch_xla.experimental.xla_quantized_matmul # noqa: F401
91 + out = torch.ops.xla.quantized_matmul(x,
92 + w_q,
93 + w_s,
94 + zero_point=None,
95 + block_size=-1,
96 + int4_weight=False,
97 + quantize_activation=True)
98 +
99 + # Explicitly capture control flow to make dynamo happy.
100 + # https://pytorch.org/docs/main/generated/exportdb/index.html#cond-branch-class-method # noqa: E501
101 + return cond(bias is None, self.no_add_bias, self.add_bias, [out, bias])
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
-38
@@ -201,44 +201,6 @@ def apply_fp8_linear(
201 return output.to(dtype=input.dtype).view(*output_shape)
202
203
204 -def apply_int8_linear(
205 - input: torch.Tensor,
206 - weight: torch.Tensor,
207 - weight_scale: torch.Tensor,
208 - input_scale: Optional[torch.Tensor] = None,
209 - input_zero_point: Optional[torch.Tensor] = None,
210 - azp_adj: Optional[torch.Tensor] = None,
211 - bias: Optional[torch.Tensor] = None,
212 -):
213 - # ops.scaled_int8_quant supports both dynamic and static quant.
214 - # * dynamic, layer.input_scale is None and x_scale computed from x.
215 - # * static, layer.input_scale is scalar and x_scale is input_scale.
216 - symmetric = azp_adj is None
217 - x_q, x_scale, x_zp = ops.scaled_int8_quant(input,
218 - input_scale,
219 - input_zero_point,
220 - symmetric=symmetric)
221 -
222 - if x_zp is not None:
223 - # Currently, static is always per-tensor and dynamic is per-token
224 - static = input_zero_point is not None
225 - azp = None if static else x_zp
226 - return ops.cutlass_scaled_mm_azp(x_q,
227 - weight,
228 - scale_a=x_scale,
229 - scale_b=weight_scale,
230 - out_dtype=input.dtype,
231 - azp_adj=azp_adj,
232 - azp=azp,
233 - bias=bias)
234 - return ops.cutlass_scaled_mm(x_q,
235 - weight,
236 - scale_a=x_scale,
237 - scale_b=weight_scale,
238 - out_dtype=input.dtype,
239 - bias=bias)
240 -
241 -
204 def normalize_e4m3fn_to_e4m3fnuz(
205 weight: torch.Tensor,
206 weight_scale: torch.Tensor,
vllm/model_executor/parameter.py
+13
@@ -6,6 +6,7 @@ from torch.nn import Parameter
6
7 from vllm.distributed import get_tensor_model_parallel_rank
8 from vllm.logger import init_logger
9 +from vllm.model_executor.utils import _make_synced_weight_loader
10
11 __all__ = [
12 "BasevLLMParameter", "PackedvLLMParameter", "PerTensorScaleParameter",
@@ -37,6 +38,18 @@ class BasevLLMParameter(Parameter):
38 :returns: a torch.nn.parameter
39 """
40
41 + # During weight loading, we often do something like:
42 + # narrowed_tensor = param.data.narrow(0, offset, len)
43 + # narrowed_tensor.copy_(real_weight)
44 + # expecting narrowed_tensor and param.data to share the same storage.
45 + # However, on TPUs, narrowed_tensor will lazily propagate to the base
46 + # tensor, which is param.data, leading to the redundant memory usage.
47 + # This sometimes causes OOM errors during model loading. To avoid this,
48 + # we sync the param tensor after its weight loader is called.
49 + from vllm.platforms import current_platform
50 + if current_platform.is_tpu():
51 + weight_loader = _make_synced_weight_loader(weight_loader)
52 +
53 self._weight_loader = weight_loader
54
55 @property
vllm/platforms/tpu.py
+3 -1
@@ -19,7 +19,9 @@ class TpuPlatform(Platform):
19 device_name: str = "tpu"
20 device_type: str = "tpu"
21 dispatch_key: str = "XLA"
22 - supported_quantization: list[str] = ["tpu_int8"]
22 + supported_quantization: list[str] = [
23 + "tpu_int8", "compressed-tensors", "compressed_tensors"
24 + ]
25
26 @classmethod
27 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend: