A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Check for _is_cuda() in compute_num_jobs (#3481)
bnellnm committed
Mar 20, 2024 at 13:06 UTC
ba8ae1d84f66dd804a97182350fee6ffcadf0faf
1 file changed
+6
-6
setup.py
+6
-6
@@ -61,12 +61,12 @@ class cmake_build_ext(build_ext):
61
except AttributeError:
62
num_jobs = os.cpu_count()
63
64
- nvcc_cuda_version = get_nvcc_cuda_version()
65
- if nvcc_cuda_version >= Version("11.2"):
66
- nvcc_threads = int(os.getenv("NVCC_THREADS", 8))
67
- num_jobs = max(1, round(num_jobs / (nvcc_threads / 4)))
68
- else:
69
- nvcc_threads = None
64
+ nvcc_threads = None
65
+ if _is_cuda():
66
+ nvcc_cuda_version = get_nvcc_cuda_version()
67
+ if nvcc_cuda_version >= Version("11.2"):
68
+ nvcc_threads = int(os.getenv("NVCC_THREADS", 8))
69
+ num_jobs = max(1, round(num_jobs / (nvcc_threads / 4)))
70
71
return num_jobs, nvcc_threads
72