A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix broken deepseek fp8 TP weights loading (#24367)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Sep 7, 2025 at 00:23 UTC
00a4e56d8dd470615f0dde2e4c996ed5564da35f
2 files changed
+5
-2
vllm/model_executor/layers/linear.py
+3
-1
@@ -262,7 +262,7 @@ class LinearBase(CustomOp):
262
self.tp_size = (get_tensor_model_parallel_world_size()
263
if not disable_tp else 1)
264
265
- def __post_init__(self):
265
+ def update_param_tp_status(self):
266
for param in self.parameters():
267
if isinstance(param, BasevLLMParameter):
268
param.tp_rank = self.tp_rank
@@ -459,6 +459,7 @@ class ColumnParallelLinear(LinearBase):
459
})
460
else:
461
self.register_parameter("bias", None)
462
+ self.update_param_tp_status()
463
464
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
465
@@ -1250,6 +1251,7 @@ class RowParallelLinear(LinearBase):
1251
})
1252
else:
1253
self.register_parameter("bias", None)
1254
+ self.update_param_tp_status()
1255
1256
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
1257
input_dim = getattr(param, "input_dim", None)
vllm/model_executor/layers/quantization/fp8.py
+2
-1
@@ -270,7 +270,8 @@ class Fp8LinearMethod(LinearMethodBase):
270
layer.weight_block_size = None
271
272
if self.block_quant:
273
- tp_size = get_tensor_model_parallel_world_size()
273
+ tp_size = getattr(layer, "tp_size",
274
+ get_tensor_model_parallel_world_size())
275
assert self.quant_config.weight_block_size is not None
276
layer.weight_block_size = self.quant_config.weight_block_size
277
block_n, block_k = (