@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 59a6abf3c9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Hotfix][CI/Build][Kernel] CUDA 11.8 does not support layernorm optimizations (#3782)

Matt Wong committed Apr 8, 2024 at 14:31 UTC 59a6abf3c99ee4fed5312d357f6ecbf857f24433
2 files changed +6 -2
cmake/utils.cmake
+2
@@ -100,6 +100,8 @@ function (get_torch_gpu_compiler_flags OUT_GPU_FLAGS GPU_LANG)
100
101 if (CUDA_VERSION VERSION_GREATER_EQUAL 11.8)
102 list(APPEND GPU_FLAGS "-DENABLE_FP8_E5M2")
103 + endif()
104 + if (CUDA_VERSION VERSION_GREATER_EQUAL 12.0)
105 list(REMOVE_ITEM GPU_FLAGS
106 "-D__CUDA_NO_HALF_OPERATORS__"
107 "-D__CUDA_NO_HALF_CONVERSIONS__"
csrc/layernorm_kernels.cu
+4 -2
@@ -59,6 +59,8 @@ __global__ void rms_norm_kernel(
59 template<typename torch_type>
60 struct _typeConvert { static constexpr bool exists = false; };
61
62 +#if defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >= 12000))
63 +// CUDA < 12.0 runs into issues with packed type conversion
64 template<>
65 struct _typeConvert<c10::Half> {
66 static constexpr bool exists = true;
@@ -85,8 +87,8 @@ struct _typeConvert<c10::BFloat16> {
87 __device__ static inline hip_type convert(float x) { return __float2bfloat16(x); }
88 __device__ static inline packed_hip_type convert(float2 x) { return __float22bfloat162_rn(x); }
89 };
88 -#endif
89 -
90 +#endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
91 +#endif // defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >= 12000))
92
93 /* Vector POD struct to generate vectorized and packed FP16/BF16 ops
94 for appropriate specializations of fused_add_rms_norm_kernel.