A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Fix `PerTensorScaleParameter` weight loading for fused models (#7376)
Dipika Sikka committed
Aug 9, 2024 at 17:23 UTC
5c6c54d67a4d7d08f1db8bcc80612d44595d1b4f
1 file changed
+14
-9
vllm/model_executor/layers/linear.py
+14
-9
@@ -14,7 +14,8 @@ from vllm.logger import init_logger
14
from vllm.model_executor.layers.quantization.base_config import (
15
QuantizationConfig, QuantizeMethodBase)
16
from vllm.model_executor.parameter import (BasevLLMParameter,
17
- PackedvLLMParameter)
17
+ PackedvLLMParameter,
18
+ PerTensorScaleParameter)
19
from vllm.model_executor.utils import set_weight_attrs
20
21
logger = init_logger(__name__)
@@ -573,11 +574,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
574
param: BasevLLMParameter,
575
loaded_weight: torch.Tensor,
576
loaded_shard_id: Optional[int] = None):
576
- param_data = param.data
577
if loaded_shard_id is None:
578
- if param.output_dim is None:
579
- assert param_data.shape == loaded_weight.shape
580
- param_data.copy_(loaded_weight)
578
+ if isinstance(param, PerTensorScaleParameter):
579
+ param.load_merged_column_weight(loaded_weight=loaded_weight,
580
+ shard_id=0)
581
+ return
582
+ elif type(param) is BasevLLMParameter:
583
+ param.load_merged_column_weight(loaded_weight=loaded_weight)
584
return
585
self._load_fused_module_from_checkpoint(param, loaded_weight)
586
return
@@ -720,11 +723,13 @@ class QKVParallelLinear(ColumnParallelLinear):
723
param: BasevLLMParameter,
724
loaded_weight: torch.Tensor,
725
loaded_shard_id: Optional[str] = None):
723
- param_data = param.data
726
if loaded_shard_id is None: # special case for certain models
725
- if param.output_dim is None:
726
- assert param_data.shape == loaded_weight.shape
727
- param_data.copy_(loaded_weight)
727
+ if isinstance(param, PerTensorScaleParameter):
728
+ param.load_merged_column_weight(loaded_weight=loaded_weight,
729
+ shard_id=0)
730
+ return
731
+ elif type(param) is BasevLLMParameter:
732
+ param.load_merged_column_weight(loaded_weight=loaded_weight)
733
return
734
self._load_fused_module_from_checkpoint(param, loaded_weight)
735
return