@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 593e79e733
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] torch.set_num_threads() in multiproc_gpu_executor (#6802)

[Bugfix] Use torch.set_num_threads() to configure parallelism in multiproc_gpu_executor (#6802) Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Jul 26, 2024 at 23:15 UTC 593e79e7337f7fd9e92b7554dabdff96769dbf15
1 file changed +19 -4
vllm/executor/multiproc_gpu_executor.py
+19 -4
@@ -6,6 +6,8 @@ import weakref
6 from functools import partial
7 from typing import Any, List, Optional
8
9 +import torch
10 +
11 from vllm.executor.distributed_gpu_executor import ( # yapf: disable
12 DistributedGPUExecutor, DistributedGPUExecutorAsync)
13 from vllm.executor.gpu_executor import create_worker
@@ -45,10 +47,23 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
47 # Disable torch async compiling which won't work with daemonic processes
48 os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
49
48 - # Set OMP_NUM_THREADS to 1 if it is not set explicitly, avoids CPU
49 - # contention amongst the shards
50 - if "OMP_NUM_THREADS" not in os.environ:
51 - os.environ["OMP_NUM_THREADS"] = "1"
50 + # Configure thread parallelism if OMP_NUM_THREADS isn't set
51 + #
52 + # Helps to avoid CPU contention. The default of spawning a thread per
53 + # core combined with multiprocessing for each GPU can have a negative
54 + # impact on performance. The contention is amplified when running in a
55 + # container where CPU limits can cause throttling.
56 + default_omp_num_threads = 1
57 + if "OMP_NUM_THREADS" not in os.environ and (
58 + current_parallelism :=
59 + torch.get_num_threads()) > default_omp_num_threads:
60 + logger.warning(
61 + "Reducing Torch parallelism from %d threads to %d to avoid "
62 + "unnecessary CPU contention. Set OMP_NUM_THREADS in the "
63 + "external environment to tune this value as needed.",
64 + current_parallelism, default_omp_num_threads)
65 + os.environ["OMP_NUM_THREADS"] = str(default_omp_num_threads)
66 + torch.set_num_threads(default_omp_num_threads)
67
68 # workaround for https://github.com/vllm-project/vllm/issues/6103
69 if world_size > 1: