@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / baee28c46c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Reorder kv dtype check to avoid nvcc not found error on AMD platform (#3104)

cloudhan committed Mar 2, 2024 at 14:34 UTC baee28c46c242b72f90d6b1211ab9d7872ab05d3
1 file changed +3 -4
vllm/config.py
+3 -4
@@ -330,15 +330,14 @@ class CacheConfig:
330 if self.cache_dtype == "auto":
331 pass
332 elif self.cache_dtype == "fp8_e5m2":
333 + if is_hip():
334 + raise NotImplementedError(
335 + "FP8_E5M2 KV Cache on AMD GPU has not been supported yet.")
336 nvcc_cuda_version = get_nvcc_cuda_version()
337 if nvcc_cuda_version and nvcc_cuda_version < Version("11.8"):
338 raise ValueError(
339 "FP8 is not supported when cuda version is lower than 11.8."
340 )
338 - device_name = torch.cuda.get_device_name()
339 - if "AMD" in device_name:
340 - raise NotImplementedError(
341 - "FP8_E5M2 KV Cache on AMD GPU has not been supported yet.")
341 logger.info(
342 "Using fp8_e5m2 data type to store kv cache. It reduces "
343 "the GPU memory footprint and boosts the performance. "