A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 05:00 UTC
[ROCm] Serving Fails on Radeon Due to AITER Dtype Import (#30952)
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
vllmellm committed
Dec 18, 2025 at 19:47 UTC
96bf50a2c0142597e83de39503ccb7cfc7732d95
1 file changed
+18
-12
vllm/_aiter_ops.py
+18
-12
@@ -24,14 +24,13 @@ def is_aiter_found() -> bool:
24
# we keep this global outside to not cause torch compile breaks.
25
IS_AITER_FOUND = is_aiter_found()
26
27
-# Can't use dtypes.fp8 directly inside an op
28
-# because it returns wrong result on gfx942.
29
-# This is a workaround to get the correct FP8 dtype.
30
-# This might because that the get_gfx() is wrapped as a custom op.
31
-if IS_AITER_FOUND:
32
- from aiter import dtypes
27
34
- AITER_FP8_DTYPE = dtypes.fp8
28
+def is_aiter_found_and_supported() -> bool:
29
+ if current_platform.is_rocm() and IS_AITER_FOUND:
30
+ from vllm.platforms.rocm import on_gfx9
31
+
32
+ return on_gfx9()
33
+ return False
34
35
36
def if_aiter_supported(func: Callable) -> Callable:
@@ -43,17 +42,24 @@ def if_aiter_supported(func: Callable) -> Callable:
42
def wrapper(*args, **kwargs):
43
# checks the platform, device arch and aiter library existence.
44
46
- if current_platform.is_rocm() and IS_AITER_FOUND:
47
- from vllm.platforms.rocm import on_gfx9
48
-
49
- if on_gfx9():
50
- return func(*args, **kwargs)
45
+ if is_aiter_found_and_supported():
46
+ return func(*args, **kwargs)
47
48
return None
49
50
return wrapper
51
52
53
+# Can't use dtypes.fp8 directly inside an op
54
+# because it returns wrong result on gfx942.
55
+# This is a workaround to get the correct FP8 dtype.
56
+# This might because that the get_gfx() is wrapped as a custom op.
57
+if is_aiter_found_and_supported():
58
+ from aiter import dtypes
59
+
60
+ AITER_FP8_DTYPE = dtypes.fp8
61
+
62
+
63
def _rocm_aiter_fused_moe_impl(
64
hidden_states: torch.Tensor,
65
w1: torch.Tensor,