A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Significant performance drop on CPUs with --num-scheduler-steps > 1 (#11794)
Jie Fu (傅杰) committed
Jan 8, 2025 at 08:15 UTC
a4e2b268568b335d8fe37f8eaaa894cec3ba9397
1 file changed
+6
vllm/engine/arg_utils.py
+6
@@ -1157,6 +1157,12 @@ class EngineArgs:
1157
if self.enable_chunked_prefill and self.pipeline_parallel_size > 1:
1158
raise ValueError("Multi-Step Chunked-Prefill is not supported "
1159
"for pipeline-parallel-size > 1")
1160
+ from vllm.platforms import current_platform
1161
+ if current_platform.is_cpu():
1162
+ logger.warning("Multi-Step (--num-scheduler-steps > 1) is "
1163
+ "currently not supported for CPUs and has been "
1164
+ "disabled.")
1165
+ self.num_scheduler_steps = 1
1166
1167
# make sure num_lookahead_slots is set the higher value depending on
1168
# if we are using speculative decoding or multi-step