@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2135cacb45
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Fix wrong multi_modal_input format for CPU runner (#5451)

Isotr0py committed Jun 13, 2024 at 07:20 UTC 2135cacb457b7daf1143c8465ab72650eaa4dd7e
1 file changed +2 -2
vllm/worker/cpu_model_runner.py
+2 -2
@@ -343,8 +343,8 @@ class CPUModelRunner:
343 "kv_caches": kv_caches,
344 "attn_metadata": attn_metadata,
345 }
346 - if self.vision_language_config:
347 - execute_model_kwargs.update({"image_input": multi_modal_input})
346 + if self.vision_language_config and multi_modal_input is not None:
347 + execute_model_kwargs.update(multi_modal_input)
348
349 hidden_states = model_executable(**execute_model_kwargs)
350