@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b185230744
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[ Misc ] Remove `fp8_shard_indexer` from Col/Row Parallel Linear (Simplify Weight Loading) (#5928)

Co-authored-by: Robert Shaw <rshaw@neuralmagic>

Robert Shaw committed Jun 28, 2024 at 13:49 UTC b185230744ae36a612527a9864c27f685acc6ef3
1 file changed +8 -20
vllm/model_executor/layers/linear.py
+8 -20
@@ -269,10 +269,6 @@ class ColumnParallelLinear(LinearBase):
269 self.register_parameter("bias", None)
270
271 def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
272 - # Special case for Fp8 scales.
273 - fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
274 - None)
275 -
272 tp_rank = get_tensor_model_parallel_rank()
273 output_dim = getattr(param, "output_dim", None)
274 param_data = param.data
@@ -281,11 +277,11 @@ class ColumnParallelLinear(LinearBase):
277 start_idx = tp_rank * shard_size
278 loaded_weight = loaded_weight.narrow(output_dim, start_idx,
279 shard_size)
284 - # Special case for Fp8 scales.
285 - elif fp8_scales_shard_indexer is not None:
286 - param_data, loaded_weight = fp8_scales_shard_indexer(param_data,
287 - loaded_weight,
288 - shard_id=0)
280 +
281 + # Special case for loading scales off disk, which often do not
282 + # have a shape (such as in the case of AutoFP8).
283 + if len(loaded_weight.shape) == 0:
284 + loaded_weight = loaded_weight.reshape(1)
285
286 assert param_data.shape == loaded_weight.shape
287 param_data.copy_(loaded_weight)
@@ -751,10 +747,6 @@ class RowParallelLinear(LinearBase):
747 self.register_parameter("bias", None)
748
749 def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
754 - # Special case for Fp8 scales.
755 - fp8_scales_shard_indexer = getattr(param, "fp8_scales_shard_indexer",
756 - None)
757 -
750 tp_rank = get_tensor_model_parallel_rank()
751 input_dim = getattr(param, "input_dim", None)
752 param_data = param.data
@@ -764,13 +756,9 @@ class RowParallelLinear(LinearBase):
756 loaded_weight = loaded_weight.narrow(input_dim, start_idx,
757 shard_size)
758
767 - # Special case for Fp8 scales.
768 - elif fp8_scales_shard_indexer is not None:
769 - param_data, loaded_weight = fp8_scales_shard_indexer(param_data,
770 - loaded_weight,
771 - shard_id=0)
772 -
773 - if fp8_scales_shard_indexer is None and len(loaded_weight.shape) == 0:
759 + # Special case for loading scales off disk, which often do not
760 + # have a shape (such as in the case of AutoFP8).
761 + if len(loaded_weight.shape) == 0:
762 loaded_weight = loaded_weight.reshape(1)
763
764 assert param_data.shape == loaded_weight.shape