A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix] Fix failing transformers dynamic module resolving with spawn multiproc method (#13403)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Feb 18, 2025 at 18:25 UTC
8cf97f86618d60b6f7ff325ef9a5e4dd359c341c
1 file changed
+17
-8
vllm/model_executor/model_loader/utils.py
+17
-8
@@ -47,22 +47,31 @@ def resolve_transformers_fallback(model_config: ModelConfig,
47
for i, arch in enumerate(architectures):
48
if arch == "TransformersModel":
49
continue
50
- custom_module = None
51
- auto_map = getattr(model_config.hf_config, "auto_map", None)
52
- if auto_map is not None and "AutoModel" in auto_map:
53
- custom_module = get_class_from_dynamic_module(
54
- model_config.hf_config.auto_map["AutoModel"],
55
- model_config.model)
50
+ auto_map: dict[str, str] = getattr(model_config.hf_config, "auto_map",
51
+ None) or dict()
52
+ # Make sure that config class is always initialized before model class,
53
+ # otherwise the model class won't be able to access the config class,
54
+ # the expected auto_map should have correct order like:
55
+ # "auto_map": {
56
+ # "AutoConfig": "<your-repo-name>--<config-name>",
57
+ # "AutoModel": "<your-repo-name>--<config-name>",
58
+ # "AutoModelFor<Task>": "<your-repo-name>--<config-name>",
59
+ # },
60
+ auto_modules = {
61
+ name: get_class_from_dynamic_module(module, model_config.model)
62
+ for name, module in sorted(auto_map.items(), key=lambda x: x[0])
63
+ }
64
+ custom_model_module = auto_modules.get("AutoModel")
65
# TODO(Isotr0py): Further clean up these raises.
66
# perhaps handled them in _ModelRegistry._raise_for_unsupported?
67
if model_config.model_impl == ModelImpl.TRANSFORMERS:
59
- if not is_transformers_impl_compatible(arch, custom_module):
68
+ if not is_transformers_impl_compatible(arch, custom_model_module):
69
raise ValueError(
70
f"The Transformers implementation of {arch} is not "
71
"compatible with vLLM.")
72
architectures[i] = "TransformersModel"
73
if model_config.model_impl == ModelImpl.AUTO:
65
- if not is_transformers_impl_compatible(arch, custom_module):
74
+ if not is_transformers_impl_compatible(arch, custom_model_module):
75
raise ValueError(
76
f"{arch} has no vLLM implementation and the Transformers "
77
"implementation is not compatible with vLLM.")