A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Fix wrong multi_modal_input format for CPU runner (#5451)
Isotr0py committed
Jun 13, 2024 at 07:20 UTC
2135cacb457b7daf1143c8465ab72650eaa4dd7e
1 file changed
+2
-2
vllm/worker/cpu_model_runner.py
+2
-2
@@ -343,8 +343,8 @@ class CPUModelRunner:
343
"kv_caches": kv_caches,
344
"attn_metadata": attn_metadata,
345
}
346
- if self.vision_language_config:
347
- execute_model_kwargs.update({"image_input": multi_modal_input})
346
+ if self.vision_language_config and multi_modal_input is not None:
347
+ execute_model_kwargs.update(multi_modal_input)
348
349
hidden_states = model_executable(**execute_model_kwargs)
350