A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Bugfix][kernels] Fix half2float conversion in gguf kernels (#15995)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Apr 5, 2025 at 00:38 UTC
230b131b54e8ad4ee9086a15c69b29b387ddb3b0
1 file changed
+5
csrc/quantization/gguf/ggml-common.h
+5
@@ -1090,6 +1090,11 @@ __device__ __forceinline__ c10::BFloat16 convert_from_half<c10::BFloat16>(half v
1090
#endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1091
}
1092
1093
+template<>
1094
+__device__ __forceinline__ float convert_from_half<float>(half val) {
1095
+ return __half2float(val);
1096
+}
1097
+
1098
#if defined(USE_ROCM)
1099
1100
#ifndef __has_builtin