A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fix nvcc not found in vlm-openai image (#2781)
zhaoyang-star committed
Feb 23, 2024 at 06:25 UTC
57f044945f25d90d1b434014b2719ba6b06fdc44
2 files changed
+10
-5
vllm/config.py
+1
-1
@@ -319,7 +319,7 @@ class CacheConfig:
319
pass
320
elif self.cache_dtype == "fp8_e5m2":
321
nvcc_cuda_version = get_nvcc_cuda_version()
322
- if nvcc_cuda_version < Version("11.8"):
322
+ if nvcc_cuda_version and nvcc_cuda_version < Version("11.8"):
323
raise ValueError(
324
"FP8 is not supported when cuda version is lower than 11.8."
325
)
vllm/utils.py
+9
-4
@@ -181,13 +181,18 @@ def set_cuda_visible_devices(device_ids: List[int]) -> None:
181
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
182
183
184
-def get_nvcc_cuda_version() -> Version:
184
+def get_nvcc_cuda_version() -> Optional[Version]:
185
cuda_home = os.environ.get('CUDA_HOME')
186
if not cuda_home:
187
cuda_home = '/usr/local/cuda'
188
- logger.info(
189
- f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'
190
- )
188
+ if os.path.isfile(cuda_home + '/bin/nvcc'):
189
+ logger.info(
190
+ f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'
191
+ )
192
+ else:
193
+ logger.warning(
194
+ f'Not found nvcc in {cuda_home}. Skip cuda version check!')
195
+ return None
196
nvcc_output = subprocess.check_output([cuda_home + "/bin/nvcc", "-V"],
197
universal_newlines=True)
198
output = nvcc_output.split()