A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[ROCM][AMD][TRITON] Halving warps number for fw_prefill to reduce spilling (#12713)
Signed-off-by: Aleksandr Malyshev <maleksan@amd.com> Co-authored-by: Aleksandr Malyshev <maleksan@amd.com>
Aleksandr Malyshev committed
Feb 4, 2025 at 19:58 UTC
64862d106efa78032702f5fa5c110ccd6d654e9a
1 file changed
+1
-1
vllm/attention/ops/prefix_prefill.py
+1
-1
@@ -11,7 +11,7 @@ from vllm.platforms import current_platform
11
12
# Static kernels parameters
13
BASE_BLOCK = 128 if current_platform.has_device_capability(80) else 64
14
-NUM_WARPS = 8
14
+NUM_WARPS = 4 if current_platform.is_rocm() else 8
15
16
# To check compatibility
17
IS_TURING = current_platform.get_device_capability() == (7, 5)