@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 565c1efa65
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[CI/Build][Bugfix] Fix CPU backend default threads num (#13077)

Li, Jiang committed Feb 12, 2025 at 00:55 UTC 565c1efa65358f43a78a52296d658651dd2b8f36
1 file changed +3
vllm/platforms/cpu.py
+3
@@ -115,6 +115,9 @@ class CpuPlatform(Platform):
115 # Environment variables for CPU executor
116 #
117
118 + # Set default threads num for OpenMP parallel
119 + os.environ["OMP_NUM_THREADS"] = str(torch.get_num_threads())
120 +
121 # Disable torch async compiling which won't work with daemonic processes
122 os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
123