A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Reorder kv dtype check to avoid nvcc not found error on AMD platform (#3104)
cloudhan committed
Mar 2, 2024 at 14:34 UTC
baee28c46c242b72f90d6b1211ab9d7872ab05d3
1 file changed
+3
-4
vllm/config.py
+3
-4
@@ -330,15 +330,14 @@ class CacheConfig:
330
if self.cache_dtype == "auto":
331
pass
332
elif self.cache_dtype == "fp8_e5m2":
333
+ if is_hip():
334
+ raise NotImplementedError(
335
+ "FP8_E5M2 KV Cache on AMD GPU has not been supported yet.")
336
nvcc_cuda_version = get_nvcc_cuda_version()
337
if nvcc_cuda_version and nvcc_cuda_version < Version("11.8"):
338
raise ValueError(
339
"FP8 is not supported when cuda version is lower than 11.8."
340
)
338
- device_name = torch.cuda.get_device_name()
339
- if "AMD" in device_name:
340
- raise NotImplementedError(
341
- "FP8_E5M2 KV Cache on AMD GPU has not been supported yet.")
341
logger.info(
342
"Using fp8_e5m2 data type to store kv cache. It reduces "
343
"the GPU memory footprint and boosts the performance. "