A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[BugFix] Fix Embedding Models with TP>1 (#5075)
Robert Shaw committed
May 28, 2024 at 08:32 UTC
9ba415588aeda8d99bda8889f90010f0d7330e89
1 file changed
+4
vllm/worker/embedding_model_runner.py
+4
@@ -79,6 +79,10 @@ class EmbeddingModelRunner(ModelRunner):
79
execute_model_kwargs.update({"image_input": multi_modal_input})
80
hidden_states = model_executable(**execute_model_kwargs)
81
82
+ # Only perform pooling in the driver worker.
83
+ if not self.is_driver_worker:
84
+ return None
85
+
86
return self.model.pooler(hidden_states=hidden_states,
87
pooling_metadata=pooling_metadata)
88