@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a4e2b26856
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Significant performance drop on CPUs with --num-scheduler-steps > 1 (#11794)

Jie Fu (傅杰) committed Jan 8, 2025 at 08:15 UTC a4e2b268568b335d8fe37f8eaaa894cec3ba9397
1 file changed +6
vllm/engine/arg_utils.py
+6
@@ -1157,6 +1157,12 @@ class EngineArgs:
1157 if self.enable_chunked_prefill and self.pipeline_parallel_size > 1:
1158 raise ValueError("Multi-Step Chunked-Prefill is not supported "
1159 "for pipeline-parallel-size > 1")
1160 + from vllm.platforms import current_platform
1161 + if current_platform.is_cpu():
1162 + logger.warning("Multi-Step (--num-scheduler-steps > 1) is "
1163 + "currently not supported for CPUs and has been "
1164 + "disabled.")
1165 + self.num_scheduler_steps = 1
1166
1167 # make sure num_lookahead_slots is set the higher value depending on
1168 # if we are using speculative decoding or multi-step