@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3a763ba0c3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[core][misc] keep compatibility for old-style classes (#10356)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 15, 2024 at 05:55 UTC 3a763ba0c3a92fdde78e855ded94f9ff29e02088
1 file changed +28 -12
vllm/model_executor/model_loader/loader.py
+28 -12
@@ -94,18 +94,34 @@ def _initialize_model(vllm_config: VllmConfig, prefix: str = "") -> nn.Module:
94 model_config = vllm_config.model_config
95 model_class, _ = get_model_architecture(model_config)
96 signatures = inspect.signature(model_class.__init__)
97 - # collect all kw-only parameters
98 - kw_only_params = [
99 - param.name for param in signatures.parameters.values()
100 - if param.kind == inspect.Parameter.KEYWORD_ONLY
101 - ]
102 - assert "vllm_config" in kw_only_params and "prefix" in kw_only_params, \
103 - ("vLLM model class must accept `vllm_config` and `prefix` as kw-only "
104 - "arguments. Possibly you have an old-style model class registered from "
105 - "out of tree and it is used for new vLLM version. "
106 - "Please check https://docs.vllm.ai/en/latest/design/class_hierarchy.html "
107 - "for the design and update the model class accordingly.")
108 - return model_class(vllm_config=vllm_config, prefix=prefix)
97 + all_params = [param.name for param in signatures.parameters.values()]
98 + if "vllm_config" in all_params and "prefix" in all_params:
99 + # new-style model class
100 + return model_class(vllm_config=vllm_config, prefix=prefix)
101 + msg = ("vLLM model class should accept `vllm_config` and `prefix` as "
102 + "input arguments. Possibly you have an old-style model class"
103 + " registered from out of tree and it is used for new vLLM version. "
104 + "Check https://docs.vllm.ai/en/latest/design/class_hierarchy.html "
105 + "for the design and update the model class accordingly.")
106 + logger.warning(msg)
107 + logger.warning(
108 + "Trying to guess the arguments for old-style model class %s",
109 + model_class)
110 + # try to be compatible with old-style model class
111 + kwargs = {}
112 + if "prefix" in all_params:
113 + kwargs["prefix"] = prefix
114 + if "config" in all_params:
115 + kwargs["config"] = model_config.hf_config
116 + if "cache_config" in all_params:
117 + kwargs["cache_config"] = vllm_config.cache_config
118 + if "quant_config" in all_params:
119 + kwargs["quant_config"] = vllm_config.quant_config
120 + if "lora_config" in all_params:
121 + kwargs["lora_config"] = vllm_config.lora_config
122 + if "scheduler_config" in all_params:
123 + kwargs["scheduler_config"] = vllm_config.scheduler_config
124 + return model_class(**kwargs)
125
126
127 class BaseModelLoader(ABC):