A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[ Bugfix ] Enabling Loading Models With Fused QKV/MLP on Disk with FP8 (#5921)
Co-authored-by: Robert Shaw <rshaw@neuralmagic>
Robert Shaw committed
Jun 28, 2024 at 14:43 UTC
2cd402e1692417b7645e4ece11bc2ab91072f47c
2 files changed
+32
-23
vllm/model_executor/layers/linear.py
+12
-2
@@ -383,8 +383,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
383
None)
384
385
if loaded_shard_id is None:
386
- # Loaded weight is already packed.
386
+ # Loaded weight is already fused on disk (qkv/mlp).
387
if output_dim is None:
388
+ # If fp8 + scale, need to send to each shard.
389
+ if fp8_scales_shard_indexer is not None:
390
+ param_data, loaded_weight = fp8_scales_shard_indexer(
391
+ param_data, loaded_weight, loaded_shard_id)
392
+
393
assert param_data.shape == loaded_weight.shape
394
param_data.copy_(loaded_weight)
395
return
@@ -567,8 +572,13 @@ class QKVParallelLinear(ColumnParallelLinear):
572
None)
573
574
if loaded_shard_id is None:
570
- # Loaded weight is already packed.
575
+ # Loaded weight is already fused on disk (qkv/mlp).
576
if output_dim is None:
577
+ # If fp8 + scale, need to send to each shard.
578
+ if fp8_scales_shard_indexer is not None:
579
+ param_data, loaded_weight = fp8_scales_shard_indexer(
580
+ param_data, loaded_weight, loaded_shard_id)
581
+
582
assert param_data.shape == loaded_weight.shape
583
param_data.copy_(loaded_weight)
584
return
vllm/model_executor/layers/quantization/fp8.py
+20
-21
@@ -98,6 +98,7 @@ class Fp8LinearMethod(LinearMethodBase):
98
"""
99
100
def __init__(self, quant_config: Fp8Config):
101
+ self.fused_module_in_checkpoint = False
102
self.quant_config = quant_config
103
self.cutlass_fp8_supported = cutlass_fp8_supported()
104
@@ -111,6 +112,7 @@ class Fp8LinearMethod(LinearMethodBase):
112
scale = Parameter(torch.empty(len(output_partition_sizes),
113
dtype=torch.float32),
114
requires_grad=False)
115
+ scale[:] = torch.finfo(torch.float8_e4m3fn).min
116
layer.register_parameter(scale_name, scale)
117
set_weight_attrs(
118
scale, {
@@ -169,11 +171,15 @@ class Fp8LinearMethod(LinearMethodBase):
171
**extra_weight_attrs)
172
173
def scales_shard_indexer(
172
- self, param: torch.Tensor, loaded_weight: torch.Tensor,
173
- shard_id: Union[str, int]) -> Tuple[torch.Tensor, torch.Tensor]:
174
+ self, param: torch.Tensor, loaded_weight: torch.Tensor,
175
+ shard_id: Optional[Union[str,
176
+ int]]) -> Tuple[torch.Tensor, torch.Tensor]:
177
qkv_idxs = {"q": 0, "k": 1, "v": 2}
178
176
- if isinstance(shard_id, int):
179
+ if shard_id is None:
180
+ shard_id = 0
181
+ self.fused_module_in_checkpoint = True
182
+ elif isinstance(shard_id, int):
183
pass
184
elif isinstance(shard_id, str):
185
if shard_id not in qkv_idxs:
@@ -205,15 +211,17 @@ class Fp8LinearMethod(LinearMethodBase):
211
# WEIGHT_SCALE / WEIGHT
212
# Loop over logical weights, requantizing with single scale.
213
max_w_scale = layer.weight_scale.max()
208
- start = 0
209
- for idx, logical_width in enumerate(layer.logical_widths):
210
- end = start + logical_width
211
- weight_dq = per_tensor_dequantize(layer.weight[start:end, :],
212
- layer.weight_scale[idx])
213
-
214
- layer.weight[start:end, :] = per_tensor_quantize(
215
- weight_dq, layer.weight_scale.max())
216
- start = end
214
+
215
+ if not self.fused_module_in_checkpoint:
216
+ start = 0
217
+ for idx, logical_width in enumerate(layer.logical_widths):
218
+ end = start + logical_width
219
+ weight_dq = per_tensor_dequantize(
220
+ layer.weight[start:end, :], layer.weight_scale[idx])
221
+
222
+ layer.weight[start:end, :] = per_tensor_quantize(
223
+ weight_dq, layer.weight_scale.max())
224
+ start = end
225
layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
226
227
# WEIGHT
@@ -227,10 +235,6 @@ class Fp8LinearMethod(LinearMethodBase):
235
if self.quant_config.activation_scheme == "dynamic":
236
layer.input_scale = None
237
elif self.quant_config.activation_scheme == "static":
230
- if not all_close_1d(layer.input_scale):
231
- raise ValueError(
232
- "All the input_scales for the logical weights of a "
233
- f"layer must be equal. But got {layer.input_scale}")
238
layer.input_scale = Parameter(layer.input_scale.max(),
239
requires_grad=False)
240
else:
@@ -317,11 +321,6 @@ class Fp8KVCacheMethod(QuantizeMethodBase):
321
del layer.kv_scale
322
323
320
-def all_close_1d(x: torch.Tensor) -> bool:
321
- assert len(x.shape) == 1
322
- return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
323
-
324
-
324
def per_tensor_quantize(tensor: torch.Tensor,
325
inv_scale: Union[float, torch.Tensor]) -> torch.Tensor:
326
finfo = torch.finfo(torch.float8_e4m3fn)