@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 955b5191c9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Misc] update fp8 to use `vLLMParameter` (#7437)

Dipika Sikka committed Aug 22, 2024 at 08:36 UTC 955b5191c966296e22cc0d9207d68665df90e2c9
4 files changed +51 -17
tests/weight_loading/models.txt
+1
@@ -15,3 +15,4 @@ compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
15 compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
16 awq, casperhansen/mixtral-instruct-awq, main
17 awq_marlin, casperhansen/mixtral-instruct-awq, main
18 +fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
\ No newline at end of file
vllm/model_executor/layers/linear.py
+13 -1
@@ -22,7 +22,7 @@ logger = init_logger(__name__)
22
23 WEIGHT_LOADER_V2_SUPPORTED = [
24 "CompressedTensorsLinearMethod", "AWQMarlinLinearMethod",
25 - "AWQLinearMethod", "GPTQMarlinLinearMethod"
25 + "AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod"
26 ]
27
28
@@ -349,6 +349,11 @@ class ColumnParallelLinear(LinearBase):
349 param_data.copy_(loaded_weight)
350
351 def weight_loader_v2(self, param: Parameter, loaded_weight: torch.Tensor):
352 + # Special case for loading scales off disk, which often do not
353 + # have a shape (such as in the case of AutoFP8).
354 + if len(loaded_weight.shape) == 0:
355 + assert loaded_weight.numel() == 1
356 + loaded_weight = loaded_weight.reshape(1)
357 param.load_column_parallel_weight(loaded_weight=loaded_weight)
358
359 def forward(self, input_):
@@ -1021,6 +1026,13 @@ class RowParallelLinear(LinearBase):
1026
1027 def weight_loader_v2(self, param: BasevLLMParameter,
1028 loaded_weight: torch.Tensor):
1029 +
1030 + # Special case for loading scales off disk, which often do not
1031 + # have a shape (such as in the case of AutoFP8).
1032 + if len(loaded_weight.shape) == 0:
1033 + assert loaded_weight.numel() == 1
1034 + loaded_weight = loaded_weight.reshape(1)
1035 +
1036 param.load_row_parallel_weight(loaded_weight=loaded_weight)
1037
1038 def forward(self, input_):
vllm/model_executor/layers/quantization/fp8.py
+29 -15
@@ -19,9 +19,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
19 is_layer_skipped)
20 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
21 all_close_1d, apply_fp8_linear, convert_to_channelwise,
22 - create_per_tensor_scale_param, cutlass_fp8_supported,
23 - normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize,
22 + cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize,
23 requantize_with_max_scale)
24 +from vllm.model_executor.parameter import (ModelWeightParameter,
25 + PerTensorScaleParameter)
26 from vllm.model_executor.utils import set_weight_attrs
27 from vllm.platforms import current_platform
28 from vllm.utils import is_hip, print_warning_once
@@ -137,6 +138,7 @@ class Fp8LinearMethod(LinearMethodBase):
138 ):
139 del input_size, output_size
140 output_size_per_partition = sum(output_partition_sizes)
141 + weight_loader = extra_weight_attrs.get("weight_loader")
142
143 layer.logical_widths = output_partition_sizes
144
@@ -148,34 +150,41 @@ class Fp8LinearMethod(LinearMethodBase):
150 weight_dtype = (torch.float8_e4m3fn
151 if self.quant_config.is_checkpoint_fp8_serialized else
152 params_dtype)
151 - weight = Parameter(torch.empty(output_size_per_partition,
152 - input_size_per_partition,
153 - dtype=weight_dtype),
154 - requires_grad=False)
153 +
154 + weight = ModelWeightParameter(data=torch.empty(
155 + output_size_per_partition,
156 + input_size_per_partition,
157 + dtype=weight_dtype),
158 + input_dim=1,
159 + output_dim=0,
160 + weight_loader=weight_loader)
161 layer.register_parameter("weight", weight)
156 - set_weight_attrs(weight, {
157 - **extra_weight_attrs,
158 - "input_dim": 1,
159 - "output_dim": 0,
160 - })
162
163 # If checkpoint is serialized fp8, load them.
164 # Otherwise, wait until process_weights_after_loading.
165 if self.quant_config.is_checkpoint_fp8_serialized:
166 # WEIGHT SCALE
166 - scale = create_per_tensor_scale_param(output_partition_sizes,
167 - **extra_weight_attrs)
167 + scale = PerTensorScaleParameter(data=torch.empty(
168 + len(output_partition_sizes), dtype=torch.float32),
169 + weight_loader=weight_loader)
170 +
171 + scale[:] = torch.finfo(torch.float32).min
172 layer.register_parameter("weight_scale", scale)
173
174 # INPUT ACTIVATION SCALE
175 if self.quant_config.activation_scheme == "static":
172 - scale = create_per_tensor_scale_param(output_partition_sizes,
173 - **extra_weight_attrs)
176 + scale = PerTensorScaleParameter(data=torch.empty(
177 + len(output_partition_sizes), dtype=torch.float32),
178 + weight_loader=weight_loader)
179 +
180 + scale[:] = torch.finfo(torch.float32).min
181 layer.register_parameter("input_scale", scale)
182 else:
183 layer.register_parameter("input_scale", None)
184
185 def process_weights_after_loading(self, layer: Module) -> None:
186 + layer.weight = torch.nn.Parameter(layer.weight.data,
187 + requires_grad=False)
188 # If checkpoint not serialized fp8, quantize the weights.
189 if not self.quant_config.is_checkpoint_fp8_serialized:
190 qweight, weight_scale = ops.scaled_fp8_quant(layer.weight,
@@ -197,6 +206,11 @@ class Fp8LinearMethod(LinearMethodBase):
206 # If checkpoint is fp8, handle that there are N scales for N
207 # shards in a fused module
208 else:
209 + layer.weight_scale = torch.nn.Parameter(layer.weight_scale.data,
210 + requires_grad=False)
211 + if self.quant_config.activation_scheme == "static":
212 + layer.input_scale = torch.nn.Parameter(layer.input_scale.data,
213 + requires_grad=False)
214 # If using marlin (w8a16), kernel uses channelwise weights,
215 # so extend the weight scales to be channelwise.
216 if self.use_marlin:
vllm/model_executor/parameter.py
+8 -1
@@ -208,10 +208,17 @@ class PerTensorScaleParameter(BasevLLMParameter):
208 if isinstance(shard_id, int):
209 return shard_id
210
211 + # if not int, assume shard_id for qkv
212 + # map to int and return
213 assert isinstance(shard_id, str)
214 assert shard_id in self.qkv_idxs
215 return self.qkv_idxs[shard_id]
216
217 + # For row parallel layers, no sharding needed
218 + # load weight into parameter as is
219 + def load_row_parallel_weight(self, *args, **kwargs):
220 + super().load_row_parallel_weight(*args, **kwargs)
221 +
222 def load_merged_column_weight(self, *args, **kwargs):
223 self._load_into_shard_id(*args, **kwargs)
224
@@ -219,7 +226,7 @@ class PerTensorScaleParameter(BasevLLMParameter):
226 self._load_into_shard_id(*args, **kwargs)
227
228 def load_column_parallel_weight(self, *args, **kwargs):
222 - self._load_into_shard_id(*args, **kwargs)
229 + super().load_row_parallel_weight(*args, **kwargs)
230
231 def _load_into_shard_id(self, loaded_weight: torch.Tensor,
232 shard_id: Union[str, int], **kwargs):