A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 19:45 UTC
[AITER] [ROCm] Fix crash when loading llama4 model with old aiter version installed, fallback to forward_native implementation (#29124)
Signed-off-by: Xiao Li <ilx@meta.com>
Xiao Li committed
Nov 20, 2025 at 17:54 UTC
ed6ae1e36a03bed4a29287163e051a7772b1d8b1
1 file changed
+13
-6
vllm/v1/sample/ops/topk_topp_sampler.py
+13
-6
@@ -60,13 +60,20 @@ class TopKTopPSampler(nn.Module):
60
logprobs_mode not in ("processed_logits", "processed_logprobs")
61
and rocm_aiter_ops.is_enabled()
62
):
63
- import aiter.ops.sampling # noqa: F401
63
+ try:
64
+ import aiter.ops.sampling # noqa: F401
65
65
- self.aiter_ops = torch.ops.aiter
66
- logger.info_once(
67
- "Using aiter sampler on ROCm (lazy import, sampling-only)."
68
- )
69
- self.forward = self.forward_hip
66
+ self.aiter_ops = torch.ops.aiter
67
+ logger.info_once(
68
+ "Using aiter sampler on ROCm (lazy import, sampling-only)."
69
+ )
70
+ self.forward = self.forward_hip
71
+ except ImportError:
72
+ logger.warning_once(
73
+ "aiter.ops.sampling is not available on ROCm. "
74
+ "Falling back to forward_native implementation."
75
+ )
76
+ self.forward = self.forward_native
77
else:
78
self.forward = self.forward_native
79