@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b4522474a3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix][Kernel] Implement acquire/release polyfill for Pascal (#8776)

sasha0552 committed Sep 25, 2024 at 04:26 UTC b4522474a32b6e0bf5573a9b6a6830cb787dfb63
2 files changed +18
csrc/custom_all_reduce.cuh
+11
@@ -131,15 +131,26 @@ DINLINE O downcast(array_t<float, O::size> val) {
131 }
132
133 static DINLINE void st_flag_release(FlagType* flag_addr, FlagType flag) {
134 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
135 asm volatile("st.release.sys.global.u32 [%1], %0;" ::"r"(flag),
136 "l"(flag_addr));
137 +#else
138 + asm volatile("membar.sys; st.volatile.global.u32 [%1], %0;" ::"r"(flag),
139 + "l"(flag_addr));
140 +#endif
141 }
142
143 static DINLINE FlagType ld_flag_acquire(FlagType* flag_addr) {
144 FlagType flag;
145 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
146 asm volatile("ld.acquire.sys.global.u32 %0, [%1];"
147 : "=r"(flag)
148 : "l"(flag_addr));
149 +#else
150 + asm volatile("ld.volatile.global.u32 %0, [%1]; membar.gl;"
151 + : "=r"(flag)
152 + : "l"(flag_addr));
153 +#endif
154 return flag;
155 }
156
csrc/custom_all_reduce_test.cu
+7
@@ -44,7 +44,14 @@
44 } while (0)
45
46 __global__ void dummy_kernel() {
47 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
48 for (int i = 0; i < 100; i++) __nanosleep(1000000); // 100ms
49 +#else
50 + for (int i = 0; i < 100; i++) {
51 + long long int start = clock64();
52 + while (clock64() - start < 150000000); // approximately 98.4ms on P40
53 + }
54 +#endif
55 }
56
57 template <typename T>