A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
Fix weight loading for some models in Transformers backend (#15544)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Mar 26, 2025 at 17:17 UTC
0af4d764d6626251923aa61adcf16c9bce488454
1 file changed
+5
-3
vllm/model_executor/models/transformers.py
+5
-3
@@ -345,9 +345,11 @@ class TransformersModel(nn.Module):
345
params_dict = dict(self.named_parameters())
346
loaded_params = set[str]()
347
for name, loaded_weight in weights:
348
- # Necessary for some models which use remote code
349
- if not name.startswith(prefix := self.model.base_model_prefix):
350
- name = maybe_prefix(prefix, name)
348
+ # Use "model" instead of base_model_prefix because
349
+ # the base model attribute in vLLM is always `model`
350
+ if not name.startswith(prefix := "model."):
351
+ name = prefix + name
352
+
353
if is_pp_missing_parameter(name, self):
354
continue
355
if name in params_dict: