A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[bugfix] fix tsts (#9959)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 2, 2024 at 15:54 UTC
3bb4befea7166850bdee3f72fe060c9c4044ba85
2 files changed
+2
-1
vllm/model_executor/model_loader/loader.py
+1
-1
@@ -150,7 +150,7 @@ def _get_model_initialization_kwargs(
150
151
152
def build_model(model_class: Type[nn.Module],
153
- vllm_config: VllmConfig,
153
+ vllm_config: Optional[VllmConfig],
154
hf_config: PretrainedConfig,
155
cache_config: Optional[CacheConfig],
156
quant_config: Optional[QuantizationConfig],
vllm/model_executor/models/utils.py
+1
@@ -252,6 +252,7 @@ def init_vllm_registered_model(
252
253
return build_model(
254
model_class,
255
+ None,
256
hf_config,
257
cache_config,
258
quant_config,