A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[ROCm][AMD] Disable auto enabling chunked prefill on ROCm (#11146)
Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>
Gregory Shtrasberg committed
Dec 13, 2024 at 00:31 UTC
00c1bde5d8cd30b14f661b11d9ad1c1d4470ddbf
1 file changed
+2
-1
vllm/engine/arg_utils.py
+2
-1
@@ -1074,7 +1074,8 @@ class EngineArgs:
1074
if (is_gpu and not use_sliding_window and not use_spec_decode
1075
and not self.enable_lora
1076
and not self.enable_prompt_adapter
1077
- and model_config.runner_type != "pooling"):
1077
+ and model_config.runner_type != "pooling"
1078
+ and not current_platform.is_rocm()):
1079
self.enable_chunked_prefill = True
1080
logger.warning(
1081
"Chunked prefill is enabled by default for models with "