A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 10:30 UTC
[CPU Backend] Improve cpu thread utilization (#42666)
Signed-off-by: Li, Tianmu <tianmu.li@intel.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Tianmu Li committed
May 18, 2026 at 03:04 UTC
cac81b6eda418fb5ca86b81197914dd02666353e
2 files changed
+4
-6
csrc/cpu/cpu_attn_impl.hpp
+1
-1
@@ -459,7 +459,7 @@ class AttentionScheduler {
459
const int64_t kv_len_per_thread =
460
(((total_kv_len / thread_num) + kv_len_alignment - 1) /
461
kv_len_alignment) *
462
- kv_len_alignment * (use_gqa ? input.num_heads_kv : input.num_heads_q);
462
+ kv_len_alignment;
463
std::vector<AttentionWorkItemGroup> workitems;
464
std::vector<ReductionWorkItemGroup> reduce_workitems;
465
workitems.reserve(1024);
vllm/utils/ompmultiprocessing.py
+3
-5
@@ -84,13 +84,11 @@ class OMPProcessManager:
84
)
85
# The time(milliseconds) that a thread should wait after
86
# completing the execution of a parallel region, before sleeping.
87
- envs_dict["KMP_BLOCKTIME"] = "1"
87
+ # A value of 5 masks thread underutilization.
88
+ # Set to 1 when debugging thread utilization issues.
89
+ envs_dict["KMP_BLOCKTIME"] = "5"
90
# Prevents the CPU to run into low performance state
91
envs_dict["KMP_TPAUSE"] = "0"
90
- # Provides fine granularity parallelism
91
- envs_dict["KMP_FORKJOIN_BARRIER_PATTERN"] = "dist,dist"
92
- envs_dict["KMP_PLAIN_BARRIER_PATTERN"] = "dist,dist"
93
- envs_dict["KMP_REDUCTION_BARRIER_PATTERN"] = "dist,dist"
92
elif self.use_gomp:
93
# set GOMP envs
94
# likes '0 1 2 ...'