@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 57f044945f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Fix nvcc not found in vlm-openai image (#2781)

zhaoyang-star committed Feb 23, 2024 at 06:25 UTC 57f044945f25d90d1b434014b2719ba6b06fdc44
2 files changed +10 -5
vllm/config.py
+1 -1
@@ -319,7 +319,7 @@ class CacheConfig:
319 pass
320 elif self.cache_dtype == "fp8_e5m2":
321 nvcc_cuda_version = get_nvcc_cuda_version()
322 - if nvcc_cuda_version < Version("11.8"):
322 + if nvcc_cuda_version and nvcc_cuda_version < Version("11.8"):
323 raise ValueError(
324 "FP8 is not supported when cuda version is lower than 11.8."
325 )
vllm/utils.py
+9 -4
@@ -181,13 +181,18 @@ def set_cuda_visible_devices(device_ids: List[int]) -> None:
181 os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
182
183
184 -def get_nvcc_cuda_version() -> Version:
184 +def get_nvcc_cuda_version() -> Optional[Version]:
185 cuda_home = os.environ.get('CUDA_HOME')
186 if not cuda_home:
187 cuda_home = '/usr/local/cuda'
188 - logger.info(
189 - f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'
190 - )
188 + if os.path.isfile(cuda_home + '/bin/nvcc'):
189 + logger.info(
190 + f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'
191 + )
192 + else:
193 + logger.warning(
194 + f'Not found nvcc in {cuda_home}. Skip cuda version check!')
195 + return None
196 nvcc_output = subprocess.check_output([cuda_home + "/bin/nvcc", "-V"],
197 universal_newlines=True)
198 output = nvcc_output.split()