A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 11:00 UTC
[bugfix] avoid NIXL_ERR_REMOTE_DISCONNECT in nixl_connector when Prefill dies (#28120)
Signed-off-by: Mathis Felardos <mathis@mistral.ai>
Mathis Felardos committed
Nov 27, 2025 at 16:32 UTC
cd007a53b4a2d7a83e35de559dc87da09302e956
1 file changed
+41
-21
vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+41
-21
@@ -1832,35 +1832,55 @@ class NixlConnectorWorker:
1832
done_req_ids: set[str] = set()
1833
for req_id, handles in list(transfers.items()):
1834
in_progress = False
1835
- for handle, _xfer_stime in handles:
1836
- xfer_state = self.nixl_wrapper.check_xfer_state(handle)
1837
- if xfer_state == "DONE":
1838
- # Get telemetry from NIXL
1839
- res = self.nixl_wrapper.get_xfer_telemetry(handle)
1840
- self.xfer_stats.record_transfer(res)
1841
- self.nixl_wrapper.release_xfer_handle(handle)
1842
- elif xfer_state == "PROC":
1843
- in_progress = True
1844
- continue
1845
- else:
1846
- # transfer failed - mark blocks as invalid
1847
- logger.error(
1848
- "NIXL transfer failed for request %s with state %s. "
1835
+ for handle, xfer_start_time in handles:
1836
+ try:
1837
+ xfer_state = self.nixl_wrapper.check_xfer_state(handle)
1838
+ if xfer_state == "DONE":
1839
+ # Get telemetry from NIXL
1840
+ res = self.nixl_wrapper.get_xfer_telemetry(handle)
1841
+ self.xfer_stats.record_transfer(res)
1842
+ self.nixl_wrapper.release_xfer_handle(handle)
1843
+ elif xfer_state == "PROC":
1844
+ in_progress = True
1845
+ continue
1846
+ else:
1847
+ logger.error(
1848
+ "NIXL transfer failed for request %s with state "
1849
+ "%s. Marking blocks as invalid.",
1850
+ req_id,
1851
+ xfer_state,
1852
+ )
1853
+ self._handle_failed_transfer(req_id, handle)
1854
+ in_progress = False
1855
+ except Exception:
1856
+ logger.exception(
1857
+ "NIXL transfer exception for request %s. "
1858
"Marking blocks as invalid.",
1859
req_id,
1851
- xfer_state,
1860
)
1853
- # mark all (logical)blocks for this request as invalid
1854
- if meta := self._recving_metadata.pop(req_id, None):
1855
- self._invalid_block_ids.update(meta.local_block_ids)
1856
- self._recving_metadata.pop(req_id, None)
1857
- self.nixl_wrapper.release_xfer_handle(handle)
1858
- self.xfer_stats.record_failed_transfer()
1861
+ self._handle_failed_transfer(req_id, handle)
1862
+ in_progress = False
1863
+
1864
if not in_progress:
1865
done_req_ids.add(req_id)
1866
del transfers[req_id]
1867
return done_req_ids
1868
1869
+ def _handle_failed_transfer(self, req_id: str, handle: int):
1870
+ """
1871
+ Handle a failed transfer by marking all (logical) blocks as invalid and
1872
+ recording the failure.
1873
+
1874
+ Args:
1875
+ req_id: The request ID.
1876
+ handle: The transfer handle.
1877
+ """
1878
+ if meta := self._recving_metadata.pop(req_id, None):
1879
+ self._invalid_block_ids.update(meta.local_block_ids)
1880
+ self._recving_metadata.pop(req_id, None)
1881
+ self.nixl_wrapper.release_xfer_handle(handle)
1882
+ self.xfer_stats.record_failed_transfer()
1883
+
1884
def start_load_kv(self, metadata: NixlConnectorMetadata):
1885
"""
1886
Start loading by triggering non-blocking nixl_xfer.