A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] torch.set_num_threads() in multiproc_gpu_executor (#6802)
[Bugfix] Use torch.set_num_threads() to configure parallelism in multiproc_gpu_executor (#6802) Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Jul 26, 2024 at 23:15 UTC
593e79e7337f7fd9e92b7554dabdff96769dbf15
1 file changed
+19
-4
vllm/executor/multiproc_gpu_executor.py
+19
-4
@@ -6,6 +6,8 @@ import weakref
6
from functools import partial
7
from typing import Any, List, Optional
8
9
+import torch
10
+
11
from vllm.executor.distributed_gpu_executor import ( # yapf: disable
12
DistributedGPUExecutor, DistributedGPUExecutorAsync)
13
from vllm.executor.gpu_executor import create_worker
@@ -45,10 +47,23 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
47
# Disable torch async compiling which won't work with daemonic processes
48
os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
49
48
- # Set OMP_NUM_THREADS to 1 if it is not set explicitly, avoids CPU
49
- # contention amongst the shards
50
- if "OMP_NUM_THREADS" not in os.environ:
51
- os.environ["OMP_NUM_THREADS"] = "1"
50
+ # Configure thread parallelism if OMP_NUM_THREADS isn't set
51
+ #
52
+ # Helps to avoid CPU contention. The default of spawning a thread per
53
+ # core combined with multiprocessing for each GPU can have a negative
54
+ # impact on performance. The contention is amplified when running in a
55
+ # container where CPU limits can cause throttling.
56
+ default_omp_num_threads = 1
57
+ if "OMP_NUM_THREADS" not in os.environ and (
58
+ current_parallelism :=
59
+ torch.get_num_threads()) > default_omp_num_threads:
60
+ logger.warning(
61
+ "Reducing Torch parallelism from %d threads to %d to avoid "
62
+ "unnecessary CPU contention. Set OMP_NUM_THREADS in the "
63
+ "external environment to tune this value as needed.",
64
+ current_parallelism, default_omp_num_threads)
65
+ os.environ["OMP_NUM_THREADS"] = str(default_omp_num_threads)
66
+ torch.set_num_threads(default_omp_num_threads)
67
68
# workaround for https://github.com/vllm-project/vllm/issues/6103
69
if world_size > 1: