A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[CI/Build][Bugfix] Fix CPU backend default threads num (#13077)
Li, Jiang committed
Feb 12, 2025 at 00:55 UTC
565c1efa65358f43a78a52296d658651dd2b8f36
1 file changed
+3
vllm/platforms/cpu.py
+3
@@ -115,6 +115,9 @@ class CpuPlatform(Platform):
115
# Environment variables for CPU executor
116
#
117
118
+ # Set default threads num for OpenMP parallel
119
+ os.environ["OMP_NUM_THREADS"] = str(torch.get_num_threads())
120
+
121
# Disable torch async compiling which won't work with daemonic processes
122
os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
123