A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[CI/Build] update default number of jobs and nvcc threads to avoid overloading the system (#3675)
youkaichao committed
Mar 27, 2024 at 21:18 UTC
3492859b687ba18db47720bcf6f07289999a2df5
1 file changed
+15
-5
setup.py
+15
-5
@@ -56,6 +56,8 @@ class cmake_build_ext(build_ext):
56
# Determine number of compilation jobs and optionally nvcc compile threads.
57
#
58
def compute_num_jobs(self):
59
+ # `num_jobs` is either the value of the MAX_JOBS environment variable
60
+ # (if defined) or the number of CPUs available.
61
num_jobs = os.environ.get("MAX_JOBS", None)
62
if num_jobs is not None:
63
num_jobs = int(num_jobs)
@@ -69,11 +71,19 @@ class cmake_build_ext(build_ext):
71
num_jobs = os.cpu_count()
72
73
nvcc_threads = None
72
- if _is_cuda():
73
- nvcc_cuda_version = get_nvcc_cuda_version()
74
- if nvcc_cuda_version >= Version("11.2"):
75
- nvcc_threads = int(os.getenv("NVCC_THREADS", 8))
76
- num_jobs = max(1, round(num_jobs / (nvcc_threads / 4)))
74
+ if _is_cuda() and get_nvcc_cuda_version() >= Version("11.2"):
75
+ # `nvcc_threads` is either the value of the NVCC_THREADS
76
+ # environment variable (if defined) or 1.
77
+ # when it is set, we reduce `num_jobs` to avoid
78
+ # overloading the system.
79
+ nvcc_threads = os.getenv("NVCC_THREADS", None)
80
+ if nvcc_threads is not None:
81
+ nvcc_threads = int(nvcc_threads)
82
+ logger.info(f"Using NVCC_THREADS={nvcc_threads} as the number"
83
+ " of nvcc threads.")
84
+ else:
85
+ nvcc_threads = 1
86
+ num_jobs = max(1, num_jobs // nvcc_threads)
87
88
return num_jobs, nvcc_threads
89