A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Kernel] Suppress mma.sp warning on CUDA 12.5 and later (#5401)
Tyler Michael Smith committed
Jun 14, 2024 at 13:02 UTC
348616ac4b72e2acc6e9a60ae94cf0f7fc29ac31
1 file changed
+42
-32
csrc/quantization/marlin/sparse/common/mma.h
+42
-32
@@ -20,6 +20,19 @@
20
21
namespace marlin_24 {
22
23
+// On CUDA earlier than 12.5, the ordered_metadata version of this instruction
24
+// is not supported. On later versions of CUDA the version without ordered
25
+// metadata results in the following warning:
26
+// | Advisory: Modifier ‘.sp::ordered_metadata’ should be used on instruction
27
+// | ‘mma’ instead of modifier ‘.sp’ as it is expected to have substantially
28
+// | reduced performance on some future architectures
29
+#if defined CUDA_VERSION && CUDA_VERSION >= 12500
30
+ #define MMA_SP_INST \
31
+ "mma.sp::ordered_metadata.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
32
+#else
33
+ #define MMA_SP_INST "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
34
+#endif
35
+
36
// m16n8k32 sparse tensor core mma instruction with fp16 inputs and fp32
37
// output/accumulation.
38
__device__ inline void mma_sp(const FragB& a_frag0, const FragB& a_frag1,
@@ -29,41 +42,38 @@ __device__ inline void mma_sp(const FragB& a_frag0, const FragB& a_frag1,
42
const uint32_t* a1 = reinterpret_cast<const uint32_t*>(&a_frag1);
43
const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
44
const uint32_t* e = reinterpret_cast<const uint32_t*>(&frag_m);
45
+
46
float* c = reinterpret_cast<float*>(&frag_c);
47
if (psel == 0) {
34
- asm volatile(
35
- "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
36
- "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
37
- "{%12,%13,%14,%15}, %16, 0x0;\n"
38
- : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
39
- : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]), "r"(b[2]),
40
- "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]),
41
- "r"(e[0]));
42
- asm volatile(
43
- "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
44
- "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
45
- "{%12,%13,%14,%15}, %16, 0x0;\n"
46
- : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
47
- : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]), "r"(b[3]),
48
- "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]), "f"(c[6]), "f"(c[7]),
49
- "r"(e[0]));
48
+ asm volatile(MMA_SP_INST
49
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
50
+ "{%12,%13,%14,%15}, %16, 0x0;\n"
51
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
52
+ : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]),
53
+ "r"(b[2]), "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]),
54
+ "f"(c[2]), "f"(c[3]), "r"(e[0]));
55
+ asm volatile(MMA_SP_INST
56
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
57
+ "{%12,%13,%14,%15}, %16, 0x0;\n"
58
+ : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
59
+ : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]),
60
+ "r"(b[3]), "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]),
61
+ "f"(c[6]), "f"(c[7]), "r"(e[0]));
62
} else {
51
- asm volatile(
52
- "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
53
- "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
54
- "{%12,%13,%14,%15}, %16, 0x1;\n"
55
- : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
56
- : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]), "r"(b[2]),
57
- "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]),
58
- "r"(e[0]));
59
- asm volatile(
60
- "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
61
- "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
62
- "{%12,%13,%14,%15}, %16, 0x1;\n"
63
- : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
64
- : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]), "r"(b[3]),
65
- "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]), "f"(c[6]), "f"(c[7]),
66
- "r"(e[0]));
63
+ asm volatile(MMA_SP_INST
64
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
65
+ "{%12,%13,%14,%15}, %16, 0x1;\n"
66
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
67
+ : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[0]),
68
+ "r"(b[2]), "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]),
69
+ "f"(c[2]), "f"(c[3]), "r"(e[0]));
70
+ asm volatile(MMA_SP_INST
71
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
72
+ "{%12,%13,%14,%15}, %16, 0x1;\n"
73
+ : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
74
+ : "r"(a0[0]), "r"(a1[0]), "r"(a0[1]), "r"(a1[1]), "r"(b[1]),
75
+ "r"(b[3]), "r"(b[5]), "r"(b[7]), "f"(c[4]), "f"(c[5]),
76
+ "f"(c[6]), "f"(c[7]), "r"(e[0]));
77
}
78
}
79