A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Misc] update fp8 to use `vLLMParameter` (#7437)
Dipika Sikka committed
Aug 22, 2024 at 08:36 UTC
955b5191c966296e22cc0d9207d68665df90e2c9
4 files changed
+51
-17
tests/weight_loading/models.txt
+1
@@ -15,3 +15,4 @@ compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
15
compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
16
awq, casperhansen/mixtral-instruct-awq, main
17
awq_marlin, casperhansen/mixtral-instruct-awq, main
18
+fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
\ No newline at end of file
vllm/model_executor/layers/linear.py
+13
-1
@@ -22,7 +22,7 @@ logger = init_logger(__name__)
22
23
WEIGHT_LOADER_V2_SUPPORTED = [
24
"CompressedTensorsLinearMethod", "AWQMarlinLinearMethod",
25
- "AWQLinearMethod", "GPTQMarlinLinearMethod"
25
+ "AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod"
26
]
27
28
@@ -349,6 +349,11 @@ class ColumnParallelLinear(LinearBase):
349
param_data.copy_(loaded_weight)
350
351
def weight_loader_v2(self, param: Parameter, loaded_weight: torch.Tensor):
352
+ # Special case for loading scales off disk, which often do not
353
+ # have a shape (such as in the case of AutoFP8).
354
+ if len(loaded_weight.shape) == 0:
355
+ assert loaded_weight.numel() == 1
356
+ loaded_weight = loaded_weight.reshape(1)
357
param.load_column_parallel_weight(loaded_weight=loaded_weight)
358
359
def forward(self, input_):
@@ -1021,6 +1026,13 @@ class RowParallelLinear(LinearBase):
1026
1027
def weight_loader_v2(self, param: BasevLLMParameter,
1028
loaded_weight: torch.Tensor):
1029
+
1030
+ # Special case for loading scales off disk, which often do not
1031
+ # have a shape (such as in the case of AutoFP8).
1032
+ if len(loaded_weight.shape) == 0:
1033
+ assert loaded_weight.numel() == 1
1034
+ loaded_weight = loaded_weight.reshape(1)
1035
+
1036
param.load_row_parallel_weight(loaded_weight=loaded_weight)
1037
1038
def forward(self, input_):
vllm/model_executor/layers/quantization/fp8.py
+29
-15
@@ -19,9 +19,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
19
is_layer_skipped)
20
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
21
all_close_1d, apply_fp8_linear, convert_to_channelwise,
22
- create_per_tensor_scale_param, cutlass_fp8_supported,
23
- normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize,
22
+ cutlass_fp8_supported, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize,
23
requantize_with_max_scale)
24
+from vllm.model_executor.parameter import (ModelWeightParameter,
25
+ PerTensorScaleParameter)
26
from vllm.model_executor.utils import set_weight_attrs
27
from vllm.platforms import current_platform
28
from vllm.utils import is_hip, print_warning_once
@@ -137,6 +138,7 @@ class Fp8LinearMethod(LinearMethodBase):
138
):
139
del input_size, output_size
140
output_size_per_partition = sum(output_partition_sizes)
141
+ weight_loader = extra_weight_attrs.get("weight_loader")
142
143
layer.logical_widths = output_partition_sizes
144
@@ -148,34 +150,41 @@ class Fp8LinearMethod(LinearMethodBase):
150
weight_dtype = (torch.float8_e4m3fn
151
if self.quant_config.is_checkpoint_fp8_serialized else
152
params_dtype)
151
- weight = Parameter(torch.empty(output_size_per_partition,
152
- input_size_per_partition,
153
- dtype=weight_dtype),
154
- requires_grad=False)
153
+
154
+ weight = ModelWeightParameter(data=torch.empty(
155
+ output_size_per_partition,
156
+ input_size_per_partition,
157
+ dtype=weight_dtype),
158
+ input_dim=1,
159
+ output_dim=0,
160
+ weight_loader=weight_loader)
161
layer.register_parameter("weight", weight)
156
- set_weight_attrs(weight, {
157
- **extra_weight_attrs,
158
- "input_dim": 1,
159
- "output_dim": 0,
160
- })
162
163
# If checkpoint is serialized fp8, load them.
164
# Otherwise, wait until process_weights_after_loading.
165
if self.quant_config.is_checkpoint_fp8_serialized:
166
# WEIGHT SCALE
166
- scale = create_per_tensor_scale_param(output_partition_sizes,
167
- **extra_weight_attrs)
167
+ scale = PerTensorScaleParameter(data=torch.empty(
168
+ len(output_partition_sizes), dtype=torch.float32),
169
+ weight_loader=weight_loader)
170
+
171
+ scale[:] = torch.finfo(torch.float32).min
172
layer.register_parameter("weight_scale", scale)
173
174
# INPUT ACTIVATION SCALE
175
if self.quant_config.activation_scheme == "static":
172
- scale = create_per_tensor_scale_param(output_partition_sizes,
173
- **extra_weight_attrs)
176
+ scale = PerTensorScaleParameter(data=torch.empty(
177
+ len(output_partition_sizes), dtype=torch.float32),
178
+ weight_loader=weight_loader)
179
+
180
+ scale[:] = torch.finfo(torch.float32).min
181
layer.register_parameter("input_scale", scale)
182
else:
183
layer.register_parameter("input_scale", None)
184
185
def process_weights_after_loading(self, layer: Module) -> None:
186
+ layer.weight = torch.nn.Parameter(layer.weight.data,
187
+ requires_grad=False)
188
# If checkpoint not serialized fp8, quantize the weights.
189
if not self.quant_config.is_checkpoint_fp8_serialized:
190
qweight, weight_scale = ops.scaled_fp8_quant(layer.weight,
@@ -197,6 +206,11 @@ class Fp8LinearMethod(LinearMethodBase):
206
# If checkpoint is fp8, handle that there are N scales for N
207
# shards in a fused module
208
else:
209
+ layer.weight_scale = torch.nn.Parameter(layer.weight_scale.data,
210
+ requires_grad=False)
211
+ if self.quant_config.activation_scheme == "static":
212
+ layer.input_scale = torch.nn.Parameter(layer.input_scale.data,
213
+ requires_grad=False)
214
# If using marlin (w8a16), kernel uses channelwise weights,
215
# so extend the weight scales to be channelwise.
216
if self.use_marlin:
vllm/model_executor/parameter.py
+8
-1
@@ -208,10 +208,17 @@ class PerTensorScaleParameter(BasevLLMParameter):
208
if isinstance(shard_id, int):
209
return shard_id
210
211
+ # if not int, assume shard_id for qkv
212
+ # map to int and return
213
assert isinstance(shard_id, str)
214
assert shard_id in self.qkv_idxs
215
return self.qkv_idxs[shard_id]
216
217
+ # For row parallel layers, no sharding needed
218
+ # load weight into parameter as is
219
+ def load_row_parallel_weight(self, *args, **kwargs):
220
+ super().load_row_parallel_weight(*args, **kwargs)
221
+
222
def load_merged_column_weight(self, *args, **kwargs):
223
self._load_into_shard_id(*args, **kwargs)
224
@@ -219,7 +226,7 @@ class PerTensorScaleParameter(BasevLLMParameter):
226
self._load_into_shard_id(*args, **kwargs)
227
228
def load_column_parallel_weight(self, *args, **kwargs):
222
- self._load_into_shard_id(*args, **kwargs)
229
+ super().load_row_parallel_weight(*args, **kwargs)
230
231
def _load_into_shard_id(self, loaded_weight: torch.Tensor,
232
shard_id: Union[str, int], **kwargs):