A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix] Fix initial DP request load imbalance (#22910)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Aug 14, 2025 at 15:20 UTC
919234fe17a701dfb5c7370df6cd1ea22202c4d7
1 file changed
+2
-4
vllm/v1/engine/core_client.py
+2
-4
@@ -965,7 +965,7 @@ class DPAsyncMPClient(AsyncMPClient):
965
966
# List of [waiting, running] pair per engine.
967
# Used only by DPLBAsyncMPClient subclass.
968
- self.lb_engines: list[list[int]] = []
968
+ self.lb_engines: list[list[int]] = [[0, 0] for _ in self.core_engines]
969
970
self.first_req_sock_addr = get_open_zmq_inproc_path()
971
self.first_req_send_socket = self.resources.first_req_send_socket = (
@@ -1121,10 +1121,8 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
1121
def get_core_engine_for_request(
1122
self, request: EngineCoreRequest) -> EngineIdentity:
1123
# Engines are in rank order.
1124
- current_counts = self.lb_engines
1124
if (eng_index := request.data_parallel_rank) is None:
1126
- if not current_counts:
1127
- return self.core_engine
1125
+ current_counts = self.lb_engines
1126
# TODO use P2C alg for larger DP sizes
1127
num_engines = len(current_counts)
1128
min_score = sys.maxsize