| 1 | /* |
| 2 | Copied from https://github.com/turboderp/exllamav2 |
| 3 | */ |
| 4 | |
| 5 | #ifndef _compat_cuh |
| 6 | #define _compat_cuh |
| 7 | |
| 8 | namespace vllm { |
| 9 | namespace gptq { |
| 10 | // atomicAdd for half types, to support CC < 7.x |
| 11 | |
| 12 | __device__ __forceinline__ void atomicAdd_half(half* address, half val) { |
| 13 | unsigned int* address_as_ui = |
| 14 | (unsigned int*)((char*)address - ((size_t)address & 2)); |
| 15 | unsigned int old = *address_as_ui; |
| 16 | unsigned int assumed; |
| 17 | |
| 18 | do { |
| 19 | assumed = old; |
| 20 | __half_raw hsum; |
| 21 | hsum.x = (size_t)address & 2 ? (old >> 16) : (old & 0xffff); |
| 22 | half tmpres = __hadd(hsum, val); |
| 23 | hsum = __half_raw(tmpres); |
| 24 | old = (size_t)address & 2 ? (old & 0xffff) | (hsum.x << 16) |
| 25 | : (old & 0xffff0000) | hsum.x; |
| 26 | old = atomicCAS(address_as_ui, assumed, old); |
| 27 | } while (assumed != old); |
| 28 | } |
| 29 | |
| 30 | // atomicAdd for half2 types |
| 31 | |
| 32 | __device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val) { |
| 33 | unsigned int* address_as_ui = (unsigned int*)address; |
| 34 | unsigned int old = *address_as_ui; |
| 35 | unsigned int assumed; |
| 36 | do { |
| 37 | assumed = old; |
| 38 | half2 old_val = *((half2*)&old); |
| 39 | half2 new_val = __hadd2(old_val, val); |
| 40 | old = atomicCAS(address_as_ui, assumed, *((unsigned int*)&new_val)); |
| 41 | } while (assumed != old); |
| 42 | } |
| 43 | |
| 44 | // |
| 45 | #if defined(__CUDA_ARCH__) || \ |
| 46 | (defined(USE_ROCM) && (HIP_VERSION_MAJOR * 100 + HIP_VERSION_MINOR) < 713) |
| 47 | #if __CUDA_ARCH__ < 700 || defined(USE_ROCM) |
| 48 | |
| 49 | __device__ __forceinline__ void atomicAdd(half* address, half val) { |
| 50 | atomicAdd_half(address, val); |
| 51 | } |
| 52 | |
| 53 | #if __CUDA_ARCH__ < 600 || defined(USE_ROCM) |
| 54 | __device__ __forceinline__ void atomicAdd(half2* address, half2 val) { |
| 55 | atomicAdd_half2(address, val); |
| 56 | } |
| 57 | #endif |
| 58 | |
| 59 | #endif |
| 60 | #endif |
| 61 | |
| 62 | } // namespace gptq |
| 63 | } // namespace vllm |
| 64 | #endif |