A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix] Fix the fp8 kv_cache check error that occurs when failing to obtain the CUDA version. (#4173)
Signed-off-by: AnyISalIn <anyisalin@gmail.com>
AnyISalIn committed
May 2, 2024 at 00:11 UTC
a88bb9b032d75aad74b2e1bd3d97b8e8a24e8b9d
1 file changed
+2
-1
vllm/config.py
+2
-1
@@ -353,7 +353,8 @@ class CacheConfig:
353
elif self.cache_dtype == "fp8":
354
if not is_hip():
355
nvcc_cuda_version = get_nvcc_cuda_version()
356
- if nvcc_cuda_version < Version("11.8"):
356
+ if nvcc_cuda_version is not None \
357
+ and nvcc_cuda_version < Version("11.8"):
358
raise ValueError(
359
"FP8 is not supported when cuda version is"
360
"lower than 11.8.")