A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix][Hardware][CPU] Fix `mm_limits` initialization for CPU backend (#7735)
Isotr0py committed
Aug 22, 2024 at 00:23 UTC
6925cdbeea0d2061e47403cce10f015d99f00ab1
1 file changed
+3
-1
vllm/worker/cpu_model_runner.py
+3
-1
@@ -113,8 +113,10 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
113
)
114
115
# Multi-modal data support
116
- self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
116
+ self.mm_registry = MULTIMODAL_REGISTRY
117
+ self.multi_modal_input_mapper = self.mm_registry \
118
.create_input_mapper(self.model_config)
119
+ self.mm_registry.init_mm_limits_per_prompt(self.model_config)
120
121
# Lazy initialization.
122
self.model: nn.Module # Set after init_Model