A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Hardware][CPU][PowerPC]Disable torch.compile() in toptopk sampling (#26987)
Signed-off-by: Akash Kaothalkar <akash.kaothalkar@ibm.com> Co-authored-by: Akash Kaothalkar <akash.kaothalkar@ibm.com>
Akash kaothalkar committed
Oct 16, 2025 at 11:06 UTC
f7d318de2b95387c29b5b9f25b4b1e79b2540f28
1 file changed
+5
-1
vllm/v1/sample/ops/topk_topp_sampler.py
+5
-1
@@ -65,7 +65,11 @@ class TopKTopPSampler(nn.Module):
65
)
66
self.forward = self.forward_native
67
elif current_platform.is_cpu():
68
- if current_platform.get_cpu_architecture() == CpuArchEnum.RISCV:
68
+ arch = current_platform.get_cpu_architecture()
69
+ # Fall back to native implementation for POWERPC and RISCV.
70
+ # On PowerPC argmax produces incorrect output with torch.compile.
71
+ # PR: https://github.com/vllm-project/vllm/pull/26987
72
+ if arch in (CpuArchEnum.RISCV, CpuArchEnum.POWERPC):
73
self.forward = self.forward_native
74
else:
75
self.forward = self.forward_cpu