A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
Improve Transformers backend model loading QoL (#17039)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Apr 23, 2025 at 15:33 UTC
8e630d680e70fe7c7a1022ed33a09e6efaad629f
1 file changed
+8
-5
vllm/model_executor/model_loader/utils.py
+8
-5
@@ -55,7 +55,10 @@ def resolve_transformers_arch(model_config: ModelConfig,
55
# "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
56
# },
57
auto_modules = {
58
- name: get_class_from_dynamic_module(module, model_config.model)
58
+ name:
59
+ get_class_from_dynamic_module(module,
60
+ model_config.model,
61
+ revision=model_config.revision)
62
for name, module in sorted(auto_map.items(), key=lambda x: x[0])
63
}
64
custom_model_module = auto_modules.get("AutoModel")
@@ -97,10 +100,10 @@ def get_model_architecture(
100
architectures = ["QuantMixtralForCausalLM"]
101
102
vllm_supported_archs = ModelRegistry.get_supported_archs()
100
- is_vllm_supported = any(arch in vllm_supported_archs
101
- for arch in architectures)
102
- if (not is_vllm_supported
103
- or model_config.model_impl == ModelImpl.TRANSFORMERS):
103
+ vllm_not_supported = not any(arch in vllm_supported_archs
104
+ for arch in architectures)
105
+ if (model_config.model_impl == ModelImpl.TRANSFORMERS or
106
+ model_config.model_impl != ModelImpl.VLLM and vllm_not_supported):
107
architectures = resolve_transformers_arch(model_config, architectures)
108
109
model_cls, arch = ModelRegistry.resolve_model_cls(architectures)