@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2cd402e169
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[ Bugfix ] Enabling Loading Models With Fused QKV/MLP on Disk with FP8 (#5921)

Co-authored-by: Robert Shaw <rshaw@neuralmagic>

Robert Shaw committed Jun 28, 2024 at 14:43 UTC 2cd402e1692417b7645e4ece11bc2ab91072f47c
2 files changed +32 -23
vllm/model_executor/layers/linear.py
+12 -2
@@ -383,8 +383,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
383 None)
384
385 if loaded_shard_id is None:
386 - # Loaded weight is already packed.
386 + # Loaded weight is already fused on disk (qkv/mlp).
387 if output_dim is None:
388 + # If fp8 + scale, need to send to each shard.
389 + if fp8_scales_shard_indexer is not None:
390 + param_data, loaded_weight = fp8_scales_shard_indexer(
391 + param_data, loaded_weight, loaded_shard_id)
392 +
393 assert param_data.shape == loaded_weight.shape
394 param_data.copy_(loaded_weight)
395 return
@@ -567,8 +572,13 @@ class QKVParallelLinear(ColumnParallelLinear):
572 None)
573
574 if loaded_shard_id is None:
570 - # Loaded weight is already packed.
575 + # Loaded weight is already fused on disk (qkv/mlp).
576 if output_dim is None:
577 + # If fp8 + scale, need to send to each shard.
578 + if fp8_scales_shard_indexer is not None:
579 + param_data, loaded_weight = fp8_scales_shard_indexer(
580 + param_data, loaded_weight, loaded_shard_id)
581 +
582 assert param_data.shape == loaded_weight.shape
583 param_data.copy_(loaded_weight)
584 return
vllm/model_executor/layers/quantization/fp8.py
+20 -21
@@ -98,6 +98,7 @@ class Fp8LinearMethod(LinearMethodBase):
98 """
99
100 def __init__(self, quant_config: Fp8Config):
101 + self.fused_module_in_checkpoint = False
102 self.quant_config = quant_config
103 self.cutlass_fp8_supported = cutlass_fp8_supported()
104
@@ -111,6 +112,7 @@ class Fp8LinearMethod(LinearMethodBase):
112 scale = Parameter(torch.empty(len(output_partition_sizes),
113 dtype=torch.float32),
114 requires_grad=False)
115 + scale[:] = torch.finfo(torch.float8_e4m3fn).min
116 layer.register_parameter(scale_name, scale)
117 set_weight_attrs(
118 scale, {
@@ -169,11 +171,15 @@ class Fp8LinearMethod(LinearMethodBase):
171 **extra_weight_attrs)
172
173 def scales_shard_indexer(
172 - self, param: torch.Tensor, loaded_weight: torch.Tensor,
173 - shard_id: Union[str, int]) -> Tuple[torch.Tensor, torch.Tensor]:
174 + self, param: torch.Tensor, loaded_weight: torch.Tensor,
175 + shard_id: Optional[Union[str,
176 + int]]) -> Tuple[torch.Tensor, torch.Tensor]:
177 qkv_idxs = {"q": 0, "k": 1, "v": 2}
178
176 - if isinstance(shard_id, int):
179 + if shard_id is None:
180 + shard_id = 0
181 + self.fused_module_in_checkpoint = True
182 + elif isinstance(shard_id, int):
183 pass
184 elif isinstance(shard_id, str):
185 if shard_id not in qkv_idxs:
@@ -205,15 +211,17 @@ class Fp8LinearMethod(LinearMethodBase):
211 # WEIGHT_SCALE / WEIGHT
212 # Loop over logical weights, requantizing with single scale.
213 max_w_scale = layer.weight_scale.max()
208 - start = 0
209 - for idx, logical_width in enumerate(layer.logical_widths):
210 - end = start + logical_width
211 - weight_dq = per_tensor_dequantize(layer.weight[start:end, :],
212 - layer.weight_scale[idx])
213 -
214 - layer.weight[start:end, :] = per_tensor_quantize(
215 - weight_dq, layer.weight_scale.max())
216 - start = end
214 +
215 + if not self.fused_module_in_checkpoint:
216 + start = 0
217 + for idx, logical_width in enumerate(layer.logical_widths):
218 + end = start + logical_width
219 + weight_dq = per_tensor_dequantize(
220 + layer.weight[start:end, :], layer.weight_scale[idx])
221 +
222 + layer.weight[start:end, :] = per_tensor_quantize(
223 + weight_dq, layer.weight_scale.max())
224 + start = end
225 layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
226
227 # WEIGHT
@@ -227,10 +235,6 @@ class Fp8LinearMethod(LinearMethodBase):
235 if self.quant_config.activation_scheme == "dynamic":
236 layer.input_scale = None
237 elif self.quant_config.activation_scheme == "static":
230 - if not all_close_1d(layer.input_scale):
231 - raise ValueError(
232 - "All the input_scales for the logical weights of a "
233 - f"layer must be equal. But got {layer.input_scale}")
238 layer.input_scale = Parameter(layer.input_scale.max(),
239 requires_grad=False)
240 else:
@@ -317,11 +321,6 @@ class Fp8KVCacheMethod(QuantizeMethodBase):
321 del layer.kv_scale
322
323
320 -def all_close_1d(x: torch.Tensor) -> bool:
321 - assert len(x.shape) == 1
322 - return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
323 -
324 -
324 def per_tensor_quantize(tensor: torch.Tensor,
325 inv_scale: Union[float, torch.Tensor]) -> torch.Tensor:
326 finfo = torch.finfo(torch.float8_e4m3fn)