@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3492859b68
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[CI/Build] update default number of jobs and nvcc threads to avoid overloading the system (#3675)

youkaichao committed Mar 27, 2024 at 21:18 UTC 3492859b687ba18db47720bcf6f07289999a2df5
1 file changed +15 -5
setup.py
+15 -5
@@ -56,6 +56,8 @@ class cmake_build_ext(build_ext):
56 # Determine number of compilation jobs and optionally nvcc compile threads.
57 #
58 def compute_num_jobs(self):
59 + # `num_jobs` is either the value of the MAX_JOBS environment variable
60 + # (if defined) or the number of CPUs available.
61 num_jobs = os.environ.get("MAX_JOBS", None)
62 if num_jobs is not None:
63 num_jobs = int(num_jobs)
@@ -69,11 +71,19 @@ class cmake_build_ext(build_ext):
71 num_jobs = os.cpu_count()
72
73 nvcc_threads = None
72 - if _is_cuda():
73 - nvcc_cuda_version = get_nvcc_cuda_version()
74 - if nvcc_cuda_version >= Version("11.2"):
75 - nvcc_threads = int(os.getenv("NVCC_THREADS", 8))
76 - num_jobs = max(1, round(num_jobs / (nvcc_threads / 4)))
74 + if _is_cuda() and get_nvcc_cuda_version() >= Version("11.2"):
75 + # `nvcc_threads` is either the value of the NVCC_THREADS
76 + # environment variable (if defined) or 1.
77 + # when it is set, we reduce `num_jobs` to avoid
78 + # overloading the system.
79 + nvcc_threads = os.getenv("NVCC_THREADS", None)
80 + if nvcc_threads is not None:
81 + nvcc_threads = int(nvcc_threads)
82 + logger.info(f"Using NVCC_THREADS={nvcc_threads} as the number"
83 + " of nvcc threads.")
84 + else:
85 + nvcc_threads = 1
86 + num_jobs = max(1, num_jobs // nvcc_threads)
87
88 return num_jobs, nvcc_threads
89