@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9ba415588a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[BugFix] Fix Embedding Models with TP>1 (#5075)

Robert Shaw committed May 28, 2024 at 08:32 UTC 9ba415588aeda8d99bda8889f90010f0d7330e89
1 file changed +4
vllm/worker/embedding_model_runner.py
+4
@@ -79,6 +79,10 @@ class EmbeddingModelRunner(ModelRunner):
79 execute_model_kwargs.update({"image_input": multi_modal_input})
80 hidden_states = model_executable(**execute_model_kwargs)
81
82 + # Only perform pooling in the driver worker.
83 + if not self.is_driver_worker:
84 + return None
85 +
86 return self.model.pooler(hidden_states=hidden_states,
87 pooling_metadata=pooling_metadata)
88