A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 11:00 UTC
[Bugfix][Quantization] Fix PerTensorScale loading with tuple shard_id in MergedColumnParallelLinear (#38517)
Signed-off-by: loukang <loukang@xiaohongshu.com>
kkyyxhll committed
Apr 7, 2026 at 23:16 UTC
98e1a43af7f1f83d978b1f39a5e7ebda093b56c1
1 file changed
+9
-1
vllm/model_executor/layers/linear.py
+9
-1
@@ -910,7 +910,15 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
910
self.validate_shard_id(loaded_shard_id)
911
if loaded_shard_id is None or isinstance(loaded_shard_id, tuple):
912
if isinstance(param, PerTensorScaleParameter):
913
- param.load_merged_column_weight(loaded_weight=loaded_weight, shard_id=0)
913
+ if isinstance(loaded_shard_id, tuple):
914
+ for idx in loaded_shard_id:
915
+ param.load_merged_column_weight(
916
+ loaded_weight=loaded_weight, shard_id=idx
917
+ )
918
+ else:
919
+ param.load_merged_column_weight(
920
+ loaded_weight=loaded_weight, shard_id=0
921
+ )
922
return
923
elif type(param) in (RowvLLMParameter, BasevLLMParameter):
924
param.load_merged_column_weight(loaded_weight=loaded_weight)