A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 06:15 UTC
[Platform] Let EPD work with non-cuda platform (#30225)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
wangxiyuan committed
Dec 18, 2025 at 14:45 UTC
a85724bd6e51af58793da365da7b70b1b72d71b9
1 file changed
+4
-1
vllm/distributed/ec_transfer/ec_connector/example_connector.py
+4
-1
@@ -73,6 +73,7 @@ class ECExampleConnector(ECConnectorBase):
73
data hashes (`mm_hash`) to encoder cache tensors.
74
kwargs (dict): Additional keyword arguments for the connector.
75
"""
76
+ from vllm.platforms import current_platform
77
78
# Get the metadata
79
metadata: ECConnectorMetadata = self._get_connector_metadata()
@@ -91,7 +92,9 @@ class ECExampleConnector(ECConnectorBase):
92
if mm_data.mm_hash in encoder_cache:
93
continue
94
filename = self._generate_filename_debug(mm_data.mm_hash)
94
- ec_cache = safetensors.torch.load_file(filename)["ec_cache"].cuda()
95
+ ec_cache = safetensors.torch.load_file(
96
+ filename, device=current_platform.device_type
97
+ )["ec_cache"]
98
encoder_cache[mm_data.mm_hash] = ec_cache
99
logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash)
100