A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[P/D] NixlConnector use cache device index for memory registration (#18969)
Signed-off-by: Piotr Tarasiewicz <ptarasiewicz@nvidia.com>
ptarasiewiczNV committed
May 31, 2025 at 17:19 UTC
8bf507d766894a8104946e026a2eeb84cbf34242
1 file changed
+2
-1
vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+2
-1
@@ -488,7 +488,8 @@ class NixlConnectorWorker:
488
for cache in cache_list:
489
base_addr = cache.data_ptr()
490
region_len = self.num_blocks * self.block_len
491
- caches_data.append((base_addr, region_len, self.rank, ""))
491
+ caches_data.append(
492
+ (base_addr, region_len, cache.device.index, ""))
493
kv_caches_base_addr.append(base_addr)
494
self.kv_caches_base_addr[self.engine_id] = kv_caches_base_addr
495
self.num_regions = len(caches_data)