A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Hardware] correct method signatures for HPU,ROCm,XPU (#18551)
Signed-off-by: Andy Xie <andy.xning@gmail.com>
Ning Xie committed
May 23, 2025 at 13:31 UTC
60cad94b86ae1cd884a327a68864926b573bc5bc
4 files changed
+16
-16
vllm/platforms/__init__.py
+4
-6
@@ -42,7 +42,6 @@ def tpu_platform_plugin() -> Optional[str]:
42
logger.debug("Confirmed TPU platform is available.")
43
except Exception as e:
44
logger.debug("TPU platform is not available because: %s", str(e))
45
- pass
45
46
return "vllm.platforms.tpu.TpuPlatform" if is_tpu else None
47
@@ -112,7 +111,6 @@ def rocm_platform_plugin() -> Optional[str]:
111
amdsmi.amdsmi_shut_down()
112
except Exception as e:
113
logger.debug("ROCm platform is not available because: %s", str(e))
115
- pass
114
115
return "vllm.platforms.rocm.RocmPlatform" if is_rocm else None
116
@@ -130,7 +128,6 @@ def hpu_platform_plugin() -> Optional[str]:
128
"habana_frameworks is not found.")
129
except Exception as e:
130
logger.debug("HPU platform is not available because: %s", str(e))
133
- pass
131
132
return "vllm.platforms.hpu.HpuPlatform" if is_hpu else None
133
@@ -148,7 +145,6 @@ def xpu_platform_plugin() -> Optional[str]:
145
logger.debug("Confirmed XPU platform is available.")
146
except Exception as e:
147
logger.debug("XPU platform is not available because: %s", str(e))
151
- pass
148
149
return "vllm.platforms.xpu.XPUPlatform" if is_xpu else None
150
@@ -170,7 +166,6 @@ def cpu_platform_plugin() -> Optional[str]:
166
167
except Exception as e:
168
logger.debug("CPU platform is not available because: %s", str(e))
173
- pass
169
170
return "vllm.platforms.cpu.CpuPlatform" if is_cpu else None
171
@@ -222,8 +217,11 @@ def resolve_current_platform_cls_qualname() -> str:
217
platform_cls_qualname = func()
218
if platform_cls_qualname is not None:
219
activated_plugins.append(name)
220
+ logger.info("Platform plugin %s loaded.", name)
221
+ logger.warning(
222
+ "Platform plugin %s function's return value is None", name)
223
except Exception:
226
- pass
224
+ logger.exception("Failed to load platform plugin %s", name)
225
226
activated_builtin_plugins = list(
227
set(activated_plugins) & set(builtin_platform_plugins.keys()))
vllm/platforms/hpu.py
+2
-2
@@ -39,8 +39,8 @@ class HpuPlatform(Platform):
39
def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
40
return True
41
42
- @staticmethod
43
- def inference_mode():
42
+ @classmethod
43
+ def inference_mode(cls):
44
return torch.no_grad()
45
46
@classmethod
vllm/platforms/rocm.py
+2
-2
@@ -217,9 +217,9 @@ class RocmPlatform(Platform):
217
major, minor = torch.cuda.get_device_capability(device_id)
218
return DeviceCapability(major=major, minor=minor)
219
220
- @staticmethod
220
+ @classmethod
221
@with_amdsmi_context
222
- def is_fully_connected(physical_device_ids: list[int]) -> bool:
222
+ def is_fully_connected(cls, physical_device_ids: list[int]) -> bool:
223
"""
224
Query if the set of gpus are fully connected by xgmi (1 hop)
225
"""
vllm/platforms/xpu.py
+8
-6
@@ -37,15 +37,17 @@ class XPUPlatform(Platform):
37
logger.info("Using IPEX attention backend.")
38
return "vllm.attention.backends.ipex_attn.IpexAttnBackend"
39
40
- @staticmethod
40
+ @classmethod
41
def get_device_capability(
42
- device_id: int = 0) -> Optional[DeviceCapability]:
42
+ cls,
43
+ device_id: int = 0,
44
+ ) -> Optional[DeviceCapability]:
45
# capacity format differs from cuda's and will cause unexpected
46
# failure, so use None directly
47
return None
48
47
- @staticmethod
48
- def get_device_name(device_id: int = 0) -> str:
49
+ @classmethod
50
+ def get_device_name(cls, device_id: int = 0) -> str:
51
return torch.xpu.get_device_name(device_id)
52
53
@classmethod
@@ -57,8 +59,8 @@ class XPUPlatform(Platform):
59
def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
60
return True
61
60
- @staticmethod
61
- def inference_mode():
62
+ @classmethod
63
+ def inference_mode(cls):
64
return torch.no_grad()
65
66
@classmethod