A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
Improve validation of TP in Transformers backend (#15540)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Mar 26, 2025 at 14:26 UTC
c091c0a58898b8a0a76e18bd6724732d80fcfc28
1 file changed
+4
-1
vllm/model_executor/models/transformers.py
+4
-1
@@ -229,7 +229,10 @@ class TransformersModel(nn.Module):
229
Apply the model's tensor parallelization plan.
230
Currently only supports linear layers.
231
"""
232
- if self.tp_size > 1 and self.config.base_model_tp_plan is None:
232
+ if not self.model.supports_tp_plan:
233
+ if self.tp_size <= 1:
234
+ return
235
+
236
raise ValueError(
237
f"{type(self.model)} does not support tensor parallel yet!")
238