@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6925cdbeea
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix][Hardware][CPU] Fix `mm_limits` initialization for CPU backend (#7735)

Isotr0py committed Aug 22, 2024 at 00:23 UTC 6925cdbeea0d2061e47403cce10f015d99f00ab1
1 file changed +3 -1
vllm/worker/cpu_model_runner.py
+3 -1
@@ -113,8 +113,10 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
113 )
114
115 # Multi-modal data support
116 - self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
116 + self.mm_registry = MULTIMODAL_REGISTRY
117 + self.multi_modal_input_mapper = self.mm_registry \
118 .create_input_mapper(self.model_config)
119 + self.mm_registry.init_mm_limits_per_prompt(self.model_config)
120
121 # Lazy initialization.
122 self.model: nn.Module # Set after init_Model