@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6d21fa1cad
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Marlin_24: Ensure the mma.sp instruction is using the ::ordered_metadata modifier (introduced with PTX 8.5) (#5136)

Alexander Matveev committed May 30, 2024 at 22:02 UTC 6d21fa1cadf1e623e302eb04c15e4927febc8cf1
1 file changed +8 -4
csrc/quantization/marlin/sparse/common/mma.h
+8 -4
@@ -32,7 +32,8 @@ __device__ inline void mma_sp(const FragB& a_frag0, const FragB& a_frag1,
32 float* c = reinterpret_cast<float*>(&frag_c);
33 if (psel == 0) {
34 asm volatile(
35 - "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
35 + "mma.sp::ordered_metadata.sync.aligned.m16n8k32.row.col.f32.f16.f16."
36 + "f32 "
37 "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
38 "{%12,%13,%14,%15}, %16, 0x0;\n"
39 : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
@@ -40,7 +41,8 @@ __device__ inline void mma_sp(const FragB& a_frag0, const FragB& a_frag1,
41 "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]),
42 "r"(e[0]));
43 asm volatile(
43 - "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
44 + "mma.sp::ordered_metadata.sync.aligned.m16n8k32.row.col.f32.f16.f16."
45 + "f32 "
46 "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
47 "{%12,%13,%14,%15}, %16, 0x0;\n"
48 : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
@@ -49,7 +51,8 @@ __device__ inline void mma_sp(const FragB& a_frag0, const FragB& a_frag1,
51 "r"(e[0]));
52 } else {
53 asm volatile(
52 - "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
54 + "mma.sp::ordered_metadata.sync.aligned.m16n8k32.row.col.f32.f16.f16."
55 + "f32 "
56 "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
57 "{%12,%13,%14,%15}, %16, 0x1;\n"
58 : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
@@ -57,7 +60,8 @@ __device__ inline void mma_sp(const FragB& a_frag0, const FragB& a_frag1,
60 "r"(b[4]), "r"(b[6]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]),
61 "r"(e[0]));
62 asm volatile(
60 - "mma.sp.sync.aligned.m16n8k32.row.col.f32.f16.f16.f32 "
63 + "mma.sp::ordered_metadata.sync.aligned.m16n8k32.row.col.f32.f16.f16."
64 + "f32 "
65 "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9, %10,%11}, "
66 "{%12,%13,%14,%15}, %16, 0x1;\n"
67 : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])