@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 166 lines 5.59 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import torch
5
6 from vllm.logger import init_logger
7 from vllm.model_executor.layers.quantization.utils.humming import (
8 apply_humming_linear,
9 convert_linear_layer_to_humming_standard,
10 get_humming_linear_compute_config,
11 prepare_humming_linear_layer_config,
12 )
13 from vllm.platforms import current_platform
14 from vllm.utils.import_utils import has_humming
15
16 from .ScaledMMLinearKernel import (
17 FP8ScaledMMLinearKernel,
18 FP8ScaledMMLinearLayerConfig,
19 Int8ScaledMMLinearKernel,
20 Int8ScaledMMLinearLayerConfig,
21 )
22
23 logger = init_logger(__name__)
24
25
26 class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
27 """Humming GEMM Kernel for FP8."""
28
29 @classmethod
30 def is_supported(
31 cls, compute_capability: int | None = None
32 ) -> tuple[bool, str | None]:
33 if not current_platform.is_cuda():
34 return False, "Humming only supported on CUDA"
35
36 if not has_humming():
37 return False, "Humming is not installed"
38
39 if not current_platform.has_device_capability(75):
40 return False, "Humming only supported on SM75+"
41
42 return True, None
43
44 @classmethod
45 def can_implement(
46 cls, config: FP8ScaledMMLinearLayerConfig
47 ) -> tuple[bool, str | None]:
48 return True, None
49
50 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
51 from vllm.utils.humming import dtypes
52
53 name_map = {"weight": "weight", "weight_scale": "weight_scale"}
54 scale_torch_dtype = self.config.weight_quant_key.scale.dtype
55 scale_dtype = dtypes.DataType.from_torch_dtype(scale_torch_dtype)
56
57 quant_config = {
58 "quant_method": "humming",
59 "dtype": "float8e4m3",
60 "scale_dtype": scale_dtype,
61 }
62
63 assert self.config.weight_quant_key.scale2 is None
64 scale_group_shape = self.config.weight_quant_key.scale.group_shape
65 if scale_group_shape.is_per_tensor():
66 quant_config["weight_scale_type"] = "tensor"
67 elif scale_group_shape.is_per_channel():
68 quant_config["weight_scale_type"] = "channel"
69 elif scale_group_shape.is_per_group():
70 quant_config["weight_scale_type"] = "group"
71 quant_config["group_size"] = scale_group_shape.col
72 else:
73 assert scale_group_shape.row > 0 and scale_group_shape.col > 0
74 quant_config["weight_scale_type"] = "block"
75 quant_config["weight_scale_group_size_n"] = scale_group_shape.row
76 quant_config["weight_scale_group_size"] = scale_group_shape.col
77
78 if hasattr(layer, "weight_scale_inv"):
79 name_map["weight_scale"] = "weight_scale_inv"
80
81 convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
82 self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
83 self.compute_config = get_humming_linear_compute_config()
84 self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
85
86 def apply_weights(
87 self,
88 layer: torch.nn.Module,
89 x: torch.Tensor,
90 bias: torch.Tensor | None = None,
91 ) -> torch.Tensor:
92 return apply_humming_linear(
93 layer,
94 x,
95 skip_bias_add=bias is None,
96 layer_config=self.layer_config,
97 compute_config=self.compute_config,
98 locks=self.locks,
99 )
100
101 def apply_scaled_mm(
102 self,
103 *,
104 A: torch.Tensor,
105 B: torch.Tensor,
106 out_dtype: torch.dtype,
107 As: torch.Tensor,
108 Bs: torch.Tensor,
109 bias: torch.Tensor | None,
110 output_shape: list,
111 ) -> torch.Tensor:
112 raise NotImplementedError(
113 "HummingFP8ScaledMMLinearKernel uses apply_weights directly"
114 )
115
116
117 class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
118 """Humming GEMM Kernel for INT8."""
119
120 @classmethod
121 def is_supported(
122 cls, compute_capability: int | None = None
123 ) -> tuple[bool, str | None]:
124 if not current_platform.is_cuda():
125 return False, "Humming only supported on CUDA"
126
127 if not has_humming():
128 return False, "Humming is not installed"
129
130 if not current_platform.has_device_capability(75):
131 return False, "Humming only supported on SM75+"
132
133 return True, None
134
135 @classmethod
136 def can_implement(
137 cls, config: Int8ScaledMMLinearLayerConfig
138 ) -> tuple[bool, str | None]:
139 return True, None
140
141 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
142 weight_name, weight_scale_name, *_ = self.layer_param_names
143 name_map = {"weight": weight_name, "weight_scale": weight_scale_name}
144 quant_config = {"quant_method": "humming", "dtype": "int8"}
145 weight = getattr(layer, weight_name)
146 weight.data = weight.data + 128
147
148 convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
149 self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
150 self.compute_config = get_humming_linear_compute_config()
151 self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
152
153 def apply_weights(
154 self,
155 layer: torch.nn.Module,
156 x: torch.Tensor,
157 bias: torch.Tensor | None = None,
158 ) -> torch.Tensor:
159 return apply_humming_linear(
160 layer,
161 x,
162 skip_bias_add=bias is None,
163 layer_config=self.layer_config,
164 compute_config=self.compute_config,
165 locks=self.locks,
166 )