A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] set OMP_NUM_THREADS to 1 by default for multiprocessing (#6109)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Travis Johnson committed
Jul 3, 2024 at 17:56 UTC
1dab9bc8a9192a6081821c3a6b6c4aee3b7912c3
1 file changed
+5
vllm/executor/multiproc_gpu_executor.py
+5
@@ -37,6 +37,11 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
37
# Disable torch async compiling which won't work with daemonic processes
38
os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
39
40
+ # Set OMP_NUM_THREADS to 1 if it is not set explicitly, avoids CPU
41
+ # contention amongst the shards
42
+ if "OMP_NUM_THREADS" not in os.environ:
43
+ os.environ["OMP_NUM_THREADS"] = "1"
44
+
45
assert world_size <= cuda_device_count_stateless(), (
46
"please set tensor_parallel_size to less than max local gpu count")
47