@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5c6c54d67a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Fix `PerTensorScaleParameter` weight loading for fused models (#7376)

Dipika Sikka committed Aug 9, 2024 at 17:23 UTC 5c6c54d67a4d7d08f1db8bcc80612d44595d1b4f
1 file changed +14 -9
vllm/model_executor/layers/linear.py
+14 -9
@@ -14,7 +14,8 @@ from vllm.logger import init_logger
14 from vllm.model_executor.layers.quantization.base_config import (
15 QuantizationConfig, QuantizeMethodBase)
16 from vllm.model_executor.parameter import (BasevLLMParameter,
17 - PackedvLLMParameter)
17 + PackedvLLMParameter,
18 + PerTensorScaleParameter)
19 from vllm.model_executor.utils import set_weight_attrs
20
21 logger = init_logger(__name__)
@@ -573,11 +574,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
574 param: BasevLLMParameter,
575 loaded_weight: torch.Tensor,
576 loaded_shard_id: Optional[int] = None):
576 - param_data = param.data
577 if loaded_shard_id is None:
578 - if param.output_dim is None:
579 - assert param_data.shape == loaded_weight.shape
580 - param_data.copy_(loaded_weight)
578 + if isinstance(param, PerTensorScaleParameter):
579 + param.load_merged_column_weight(loaded_weight=loaded_weight,
580 + shard_id=0)
581 + return
582 + elif type(param) is BasevLLMParameter:
583 + param.load_merged_column_weight(loaded_weight=loaded_weight)
584 return
585 self._load_fused_module_from_checkpoint(param, loaded_weight)
586 return
@@ -720,11 +723,13 @@ class QKVParallelLinear(ColumnParallelLinear):
723 param: BasevLLMParameter,
724 loaded_weight: torch.Tensor,
725 loaded_shard_id: Optional[str] = None):
723 - param_data = param.data
726 if loaded_shard_id is None: # special case for certain models
725 - if param.output_dim is None:
726 - assert param_data.shape == loaded_weight.shape
727 - param_data.copy_(loaded_weight)
727 + if isinstance(param, PerTensorScaleParameter):
728 + param.load_merged_column_weight(loaded_weight=loaded_weight,
729 + shard_id=0)
730 + return
731 + elif type(param) is BasevLLMParameter:
732 + param.load_merged_column_weight(loaded_weight=loaded_weight)
733 return
734 self._load_fused_module_from_checkpoint(param, loaded_weight)
735 return