@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9b44a7d926
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[P/D] NIXL Updates (#25844)

Signed-off-by: Sage Moore <sage@neuralmagic.com> Signed-off-by: simon-mo <simon.mo@hey.com> Signed-off-by: rentianyue-jk <rentianyue-jk@360shuke.com> Signed-off-by: Russell Bryant <rbryant@redhat.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Chenheli Hua <huachenheli@outlook.com> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Signed-off-by: NickLucche <nlucches@redhat.com> Signed-off-by: Roger Wang <hey@rogerw.io> Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Sage Moore <sage@neuralmagic.com> Co-authored-by: Russell Bryant <rbryant@redhat.com> Co-authored-by: rentianyue-jk <rentianyue-jk@360shuke.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Chenheli Hua <huachenheli@outlook.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Robert Shaw <robshaw@redhat.com>

Robert Shaw committed Sep 29, 2025 at 00:46 UTC 9b44a7d9265b78c7b72c20e0698ede17d62de5e8
2 files changed +11 -3
vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+5 -2
@@ -474,8 +474,11 @@ class NixlConnectorWorker:
474 "backends", ["UCX"])
475 # Agent.
476 non_ucx_backends = [b for b in self.nixl_backends if b != "UCX"]
477 - config = nixl_agent_config(backends=self.nixl_backends) if len(
478 - non_ucx_backends) > 0 and nixl_agent_config is not None else None
477 + if nixl_agent_config is None:
478 + config = None
479 + else:
480 + config = nixl_agent_config(backends=self.nixl_backends) if len(
481 + non_ucx_backends) > 0 else nixl_agent_config(num_threads=8)
482
483 self.nixl_wrapper = NixlWrapper(str(uuid.uuid4()), config)
484 # Map of engine_id -> {rank0: agent_name0, rank1: agent_name1..}.
vllm/v1/core/sched/scheduler.py
+6 -1
@@ -1290,4 +1290,9 @@ class Scheduler(SchedulerInterface):
1290 self.finished_recving_kv_req_ids.add(req_id)
1291 for req_id in (kv_connector_output.finished_sending or ()):
1292 logger.debug("Finished sending KV transfer for request %s", req_id)
1293 - self._free_blocks(self.requests[req_id])
1293 + if req_id not in self.requests:
1294 + logger.warning(
1295 + "Got finished sending KV transfer for request %s,"
1296 + "but the request is already freed.", req_id)
1297 + else:
1298 + self._free_blocks(self.requests[req_id])