| 1 | /*************************************************************************************************** |
| 2 | * Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights |
| 3 | *reserved. SPDX-License-Identifier: BSD-3-Clause |
| 4 | * |
| 5 | * Redistribution and use in source and binary forms, with or without |
| 6 | * modification, are permitted provided that the following conditions are met: |
| 7 | * |
| 8 | * 1. Redistributions of source code must retain the above copyright notice, |
| 9 | *this list of conditions and the following disclaimer. |
| 10 | * |
| 11 | * 2. Redistributions in binary form must reproduce the above copyright notice, |
| 12 | * this list of conditions and the following disclaimer in the documentation |
| 13 | * and/or other materials provided with the distribution. |
| 14 | * |
| 15 | * 3. Neither the name of the copyright holder nor the names of its |
| 16 | * contributors may be used to endorse or promote products derived from |
| 17 | * this software without specific prior written permission. |
| 18 | * |
| 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| 20 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 21 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| 22 | *ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE |
| 23 | *LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 24 | *CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 25 | *SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 26 | *INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| 27 | *CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 28 | *ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| 29 | *POSSIBILITY OF SUCH DAMAGE. |
| 30 | * |
| 31 | **************************************************************************************************/ |
| 32 | |
| 33 | // |
| 34 | // This file is a modified excerpt of |
| 35 | // include/cutlass/epilogue/fusion/sm90_visitor_load_tma_warpspecialized.hpp |
| 36 | // from https://github.com/NVIDIA/cutlass v3.5.0 |
| 37 | // It has been modified to support either row/column or scalar broadcasting |
| 38 | // where the tensor being loaded from is always passed in via a device pointer. |
| 39 | // This lets one compiled kernel handle all cases of per-tensor or |
| 40 | // per-channel/per-token quantization. |
| 41 | // |
| 42 | // This interface also allows the scales to be passed in as tensors that |
| 43 | // consistently reside on the device, which avoids an issue with a previous |
| 44 | // implementation where scalars needed to be on the CPU since they |
| 45 | // were passed in via float values. This created a potential performance hazard |
| 46 | // if scales were initially on the device, and caused torch.compile graphs |
| 47 | // breaks when moving scales to the CPU. |
| 48 | // |
| 49 | #pragma once |
| 50 | |
| 51 | // Turn off clang-format for the entire file to keep it close to upstream |
| 52 | // clang-format off |
| 53 | |
| 54 | #include "cutlass/cutlass.h" |
| 55 | #include "cutlass/arch/barrier.h" |
| 56 | |
| 57 | #include "cute/tensor.hpp" |
| 58 | #include "cutlass/epilogue/fusion/sm90_visitor_tma_warpspecialized.hpp" |
| 59 | |
| 60 | namespace cutlass::epilogue::fusion { |
| 61 | |
| 62 | using namespace cute; |
| 63 | using namespace detail; |
| 64 | |
| 65 | // Row vector broadcast |
| 66 | template< |
| 67 | int Stages, |
| 68 | class CtaTileShapeMNK, |
| 69 | class Element, |
| 70 | class StrideMNL = Stride<_0,_1,_0>, |
| 71 | int Alignment = 128 / sizeof_bits_v<Element> |
| 72 | > |
| 73 | struct Sm90RowOrScalarBroadcastArray { |
| 74 | static_assert(Stages == 0, "Row broadcast doesn't support smem usage"); |
| 75 | static_assert(is_static_v<decltype(take<0,2>(StrideMNL{}))>); // batch stride can be dynamic or static |
| 76 | static_assert(take<0,2>(StrideMNL{}) == Stride<_0,_1>{}); |
| 77 | |
| 78 | struct SharedStorage { |
| 79 | array_aligned<Element, size<1>(CtaTileShapeMNK{})> smem; |
| 80 | }; |
| 81 | |
| 82 | // This struct has been modified to have a bool indicating that ptr_row is a |
| 83 | // scalar that must be broadcast, instead of containing a scalar that is |
| 84 | // valid if ptr_row is null. |
| 85 | struct Arguments { |
| 86 | const Element* const* ptr_row_array = nullptr; |
| 87 | bool row_broadcast = true; |
| 88 | StrideMNL dRow = {}; |
| 89 | }; |
| 90 | |
| 91 | using Params = Arguments; |
| 92 | |
| 93 | template <class ProblemShape> |
| 94 | static constexpr Params |
| 95 | to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) { |
| 96 | return args; |
| 97 | } |
| 98 | |
| 99 | template <class ProblemShape> |
| 100 | static bool |
| 101 | can_implement(ProblemShape const& problem_shape, Arguments const& args) { |
| 102 | return true; |
| 103 | } |
| 104 | |
| 105 | template <class ProblemShape> |
| 106 | static size_t |
| 107 | get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) { |
| 108 | return 0; |
| 109 | } |
| 110 | |
| 111 | template <class ProblemShape> |
| 112 | static cutlass::Status |
| 113 | initialize_workspace(ProblemShape const& problem_shape, Arguments const& args, void* workspace, cudaStream_t stream, |
| 114 | CudaHostAdapter* cuda_adapter = nullptr) { |
| 115 | return cutlass::Status::kSuccess; |
| 116 | } |
| 117 | |
| 118 | CUTLASS_HOST_DEVICE |
| 119 | Sm90RowOrScalarBroadcastArray() { } |
| 120 | |
| 121 | CUTLASS_HOST_DEVICE |
| 122 | Sm90RowOrScalarBroadcastArray(Params const& params, SharedStorage const& shared_storage) |
| 123 | : params(params) |
| 124 | , smem(const_cast<Element*>(shared_storage.smem.data())) { } |
| 125 | |
| 126 | Params params; |
| 127 | Element *smem = nullptr; |
| 128 | |
| 129 | CUTLASS_DEVICE bool |
| 130 | is_producer_load_needed() const { |
| 131 | return false; |
| 132 | } |
| 133 | |
| 134 | CUTLASS_DEVICE bool |
| 135 | is_C_load_needed() const { |
| 136 | return false; |
| 137 | } |
| 138 | |
| 139 | CUTLASS_DEVICE bool |
| 140 | is_zero() const { |
| 141 | return (!params.row_broadcast && *(params.ptr_row_array[group]) == Element(0)); |
| 142 | } |
| 143 | |
| 144 | template <class... Args> |
| 145 | CUTLASS_DEVICE auto |
| 146 | get_producer_load_callbacks(ProducerLoadArgs<Args...> const& args) { |
| 147 | return EmptyProducerLoadCallbacks{}; |
| 148 | } |
| 149 | |
| 150 | template <class GS_GTensor, class GS_STensor, class GS_CTensor, class Tiled_G2S, class SR_STensor, class SR_RTensor, class CTensor, class ThrResidue, class ThrNum> |
| 151 | struct ConsumerStoreCallbacks : EmptyConsumerStoreCallbacks { |
| 152 | CUTLASS_DEVICE |
| 153 | ConsumerStoreCallbacks( |
| 154 | GS_GTensor tGS_gRow_, GS_STensor tGS_sRow_, |
| 155 | GS_CTensor tGS_cRow_, Tiled_G2S tiled_g2s_, |
| 156 | SR_STensor tSR_sRow_, SR_RTensor tSR_rRow_, |
| 157 | CTensor tCcRow_, ThrResidue residue_tCcRow_, ThrNum thr_num_, |
| 158 | int group, Params const& params_) |
| 159 | : tGS_gRow(tGS_gRow_) |
| 160 | , tGS_sRow(tGS_sRow_) |
| 161 | , tGS_cRow(tGS_cRow_) |
| 162 | , tiled_G2S(tiled_g2s_) |
| 163 | , tSR_sRow(tSR_sRow_) |
| 164 | , tSR_rRow(tSR_rRow_) |
| 165 | , tCcRow(tCcRow_) |
| 166 | , residue_tCcRow(residue_tCcRow_) |
| 167 | , group(group) |
| 168 | , params(params_) {} |
| 169 | |
| 170 | GS_GTensor tGS_gRow; // (CPY,CPY_M,CPY_N) |
| 171 | GS_STensor tGS_sRow; // (CPY,CPY_M,CPY_N) |
| 172 | GS_CTensor tGS_cRow; // (CPY,CPY_M,CPY_N) |
| 173 | Tiled_G2S tiled_G2S; |
| 174 | |
| 175 | SR_STensor tSR_sRow; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) |
| 176 | SR_RTensor tSR_rRow; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) |
| 177 | |
| 178 | CTensor tCcRow; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) |
| 179 | ThrResidue residue_tCcRow; // (m, n) |
| 180 | ThrNum thr_num; |
| 181 | int group; |
| 182 | Params const& params; |
| 183 | |
| 184 | CUTLASS_DEVICE void |
| 185 | begin() { |
| 186 | if (!params.row_broadcast) { |
| 187 | fill(tSR_rRow, *(params.ptr_row_array[group])); |
| 188 | return; |
| 189 | } |
| 190 | |
| 191 | auto synchronize = [&] () { cutlass::arch::NamedBarrier::sync(thr_num, cutlass::arch::ReservedNamedBarriers::EpilogueBarrier); }; |
| 192 | cute::Tensor tGS_gRow_flt = filter_zeros(tGS_gRow); |
| 193 | cute::Tensor tGS_sRow_flt = filter_zeros(tGS_sRow); |
| 194 | cute::Tensor tGS_cRow_flt = make_tensor(tGS_cRow.data(), make_layout(tGS_gRow_flt.shape(), tGS_cRow.stride())); |
| 195 | |
| 196 | for (int i = 0; i < size(tGS_gRow_flt); ++i) { |
| 197 | if (get<1>(tGS_cRow_flt(i)) >= size<1>(CtaTileShapeMNK{})) { |
| 198 | continue; // OOB of SMEM, |
| 199 | } |
| 200 | if (elem_less(tGS_cRow_flt(i), make_coord(get<0>(residue_tCcRow), get<1>(residue_tCcRow)))) { |
| 201 | tGS_sRow_flt(i) = tGS_gRow_flt(i); |
| 202 | } |
| 203 | else { |
| 204 | tGS_sRow_flt(i) = Element(0); // Set to Zero when OOB so LDS could be issue without any preds. |
| 205 | } |
| 206 | } |
| 207 | synchronize(); |
| 208 | } |
| 209 | |
| 210 | CUTLASS_DEVICE void |
| 211 | begin_loop(int epi_m, int epi_n) { |
| 212 | if (epi_m == 0) { // Assumes M-major subtile loop |
| 213 | if (!params.row_broadcast) return; // Do not issue LDS when row is scalar |
| 214 | cute::Tensor tSR_sRow_flt = filter_zeros(tSR_sRow(_,_,_,epi_m,epi_n)); |
| 215 | cute::Tensor tSR_rRow_flt = filter_zeros(tSR_rRow); |
| 216 | copy(tSR_sRow_flt, tSR_rRow_flt); |
| 217 | } |
| 218 | } |
| 219 | |
| 220 | template <typename ElementAccumulator, int FragmentSize> |
| 221 | CUTLASS_DEVICE Array<Element, FragmentSize> |
| 222 | visit(Array<ElementAccumulator, FragmentSize> const& frg_acc, int epi_v, int epi_m, int epi_n) { |
| 223 | Array<Element, FragmentSize> frg_row; |
| 224 | |
| 225 | CUTLASS_PRAGMA_UNROLL |
| 226 | for (int i = 0; i < FragmentSize; ++i) { |
| 227 | frg_row[i] = tSR_rRow(epi_v * FragmentSize + i); |
| 228 | } |
| 229 | |
| 230 | return frg_row; |
| 231 | } |
| 232 | }; |
| 233 | |
| 234 | template < |
| 235 | bool ReferenceSrc, // do register tensors reference the src or dst layout of the tiled copy |
| 236 | class... Args |
| 237 | > |
| 238 | CUTLASS_DEVICE auto |
| 239 | get_consumer_store_callbacks(ConsumerStoreArgs<Args...> const& args) { |
| 240 | auto [M, N, K, L] = args.problem_shape_mnkl; |
| 241 | auto [m, n, k, l] = args.tile_coord_mnkl; |
| 242 | using ThreadCount = decltype(size(args.tiled_copy)); |
| 243 | |
| 244 | cute::Tensor mRow = make_tensor(make_gmem_ptr(params.ptr_row_array[l]), make_shape(M,N,1), params.dRow); |
| 245 | cute::Tensor gRow = local_tile(mRow(_,_,l), take<0,2>(args.tile_shape_mnk), make_coord(m, n)); // (CTA_M, CTA_N) |
| 246 | cute::Tensor sRow = make_tensor(make_smem_ptr(smem), |
| 247 | make_shape(size<0>(CtaTileShapeMNK{}), size<1>(CtaTileShapeMNK{})), make_shape(_0{}, _1{})); // (CTA_M, CTA_N) |
| 248 | //// G2S: Gmem to Smem |
| 249 | auto tiled_g2s = make_tiled_copy(Copy_Atom<DefaultCopy, Element>{}, |
| 250 | Layout< Shape<_1, ThreadCount>, |
| 251 | Stride<_0, _1>>{}, |
| 252 | Layout<_1>{}); |
| 253 | auto thr_g2s = tiled_g2s.get_slice(args.thread_idx); |
| 254 | cute::Tensor tGS_gRow = thr_g2s.partition_S(gRow); |
| 255 | cute::Tensor tGS_sRow = thr_g2s.partition_D(sRow); |
| 256 | |
| 257 | //// G2S: Coord |
| 258 | auto cRow = make_identity_tensor(make_shape(size<0>(CtaTileShapeMNK{}), size<1>(CtaTileShapeMNK{}))); |
| 259 | cute::Tensor tGS_cRow = thr_g2s.partition_S(cRow); |
| 260 | |
| 261 | //// S2R: Smem to Reg |
| 262 | cute::Tensor tSR_sRow = sm90_partition_for_epilogue<ReferenceSrc>(sRow, args.epi_tile, args.tiled_copy, args.thread_idx); |
| 263 | cute::Tensor tSR_rRow = make_tensor_like(take<0,3>(tSR_sRow)); // (CPY,CPY_M,CPY_N) |
| 264 | |
| 265 | return ConsumerStoreCallbacks<decltype(tGS_gRow), decltype(tGS_sRow), decltype(tGS_cRow), decltype(tiled_g2s), decltype(tSR_sRow), decltype(tSR_rRow), decltype(args.tCcD), decltype(args.residue_cD), ThreadCount>( |
| 266 | tGS_gRow, |
| 267 | tGS_sRow, |
| 268 | tGS_cRow, tiled_g2s, |
| 269 | tSR_sRow, |
| 270 | tSR_rRow, |
| 271 | args.tCcD, |
| 272 | args.residue_cD, |
| 273 | ThreadCount{}, |
| 274 | l, |
| 275 | params); |
| 276 | } |
| 277 | }; |
| 278 | |
| 279 | ///////////////////////////////////////////////////////////////////////////////////////////////// |
| 280 | |
| 281 | // Column vector broadcast |
| 282 | template< |
| 283 | int Stages, |
| 284 | class CtaTileShapeMNK, |
| 285 | class Element, |
| 286 | class StrideMNL = Stride<_1,_0,_0>, |
| 287 | int Alignment = 128 / sizeof_bits_v<Element> |
| 288 | > |
| 289 | struct Sm90ColOrScalarBroadcastArray { |
| 290 | static_assert(Stages == 0, "Column broadcast doesn't support smem usage yet"); |
| 291 | static_assert(Alignment * sizeof_bits_v<Element> % 128 == 0, "sub-16B alignment not supported yet"); |
| 292 | static_assert( |
| 293 | (cute::is_same_v<StrideMNL, Stride<_1,_0, _0>>) || // col vector broadcast, e.g. per-row alpha/bias |
| 294 | (cute::is_same_v<StrideMNL, Stride<_1,_0,int>>)); // batched col vector broadcast, e.g. batched per-row bias |
| 295 | |
| 296 | // Accumulator distributes col elements evenly amongst threads so we can just directly load from gmem |
| 297 | struct SharedStorage { }; |
| 298 | |
| 299 | // This struct has been modified to have a bool indicating that ptr_col is a |
| 300 | // scalar that must be broadcast, instead of containing a scalar that is |
| 301 | // valid if ptr_col is null. |
| 302 | struct Arguments { |
| 303 | const Element* const* ptr_col_array = nullptr; |
| 304 | bool col_broadcast = true; |
| 305 | StrideMNL dCol = {}; |
| 306 | }; |
| 307 | |
| 308 | using Params = Arguments; |
| 309 | |
| 310 | template <class ProblemShape> |
| 311 | static constexpr Params |
| 312 | to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) { |
| 313 | return args; |
| 314 | } |
| 315 | |
| 316 | template <class ProblemShape> |
| 317 | static bool |
| 318 | can_implement(ProblemShape const& problem_shape, Arguments const& args) { |
| 319 | return true; |
| 320 | } |
| 321 | |
| 322 | template <class ProblemShape> |
| 323 | static size_t |
| 324 | get_workspace_size(ProblemShape const& problem_shape, Arguments const& args) { |
| 325 | return 0; |
| 326 | } |
| 327 | |
| 328 | template <class ProblemShape> |
| 329 | static cutlass::Status |
| 330 | initialize_workspace(ProblemShape const& problem_shape, Arguments const& args, void* workspace, cudaStream_t stream, |
| 331 | CudaHostAdapter* cuda_adapter = nullptr) { |
| 332 | return cutlass::Status::kSuccess; |
| 333 | } |
| 334 | |
| 335 | CUTLASS_DEVICE bool |
| 336 | is_producer_load_needed() const { |
| 337 | return false; |
| 338 | } |
| 339 | |
| 340 | CUTLASS_DEVICE bool |
| 341 | is_C_load_needed() const { |
| 342 | return false; |
| 343 | } |
| 344 | |
| 345 | CUTLASS_DEVICE bool |
| 346 | is_zero() const { |
| 347 | return (!params.col_broadcast && *(params.ptr_col_array[group]) == Element(0)); |
| 348 | } |
| 349 | |
| 350 | CUTLASS_HOST_DEVICE |
| 351 | Sm90ColOrScalarBroadcastArray() { } |
| 352 | |
| 353 | CUTLASS_HOST_DEVICE |
| 354 | Sm90ColOrScalarBroadcastArray(Params const& params, SharedStorage const& shared_storage) |
| 355 | : params(params) { } |
| 356 | |
| 357 | Params params; |
| 358 | |
| 359 | template <class... Args> |
| 360 | CUTLASS_DEVICE auto |
| 361 | get_producer_load_callbacks(ProducerLoadArgs<Args...> const& args) { |
| 362 | return EmptyProducerLoadCallbacks{}; |
| 363 | } |
| 364 | |
| 365 | template<class GTensor, class RTensor, class CTensor, class ProblemShape> |
| 366 | struct ConsumerStoreCallbacks : EmptyConsumerStoreCallbacks { |
| 367 | CUTLASS_DEVICE |
| 368 | ConsumerStoreCallbacks( |
| 369 | GTensor&& tCgCol, |
| 370 | RTensor&& tCrCol, |
| 371 | CTensor&& tCcCol, |
| 372 | ProblemShape problem_shape, |
| 373 | int group, |
| 374 | Params const& params |
| 375 | ): |
| 376 | tCgCol(cute::forward<GTensor>(tCgCol)), |
| 377 | tCrCol(cute::forward<RTensor>(tCrCol)), |
| 378 | tCcCol(cute::forward<CTensor>(tCcCol)), |
| 379 | m(get<0>(problem_shape)), |
| 380 | group(group), |
| 381 | params(params) {} |
| 382 | |
| 383 | GTensor tCgCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) |
| 384 | RTensor tCrCol; |
| 385 | CTensor tCcCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) |
| 386 | Params const& params; |
| 387 | int m; |
| 388 | int group; |
| 389 | |
| 390 | CUTLASS_DEVICE void |
| 391 | begin() { |
| 392 | if (!params.col_broadcast) { |
| 393 | fill(tCrCol, *(params.ptr_col_array[group])); |
| 394 | return; |
| 395 | } |
| 396 | |
| 397 | // tCgCol has layout (CPY,CPY_M,CPY_N,EPI_M,EPI_N) where CPY_N and |
| 398 | // EPI_N are stride-0 for the column broadcast. Slice those modes at |
| 399 | // index 0 to avoid redundant copies AND ensure pred/data consistency |
| 400 | static_assert(decltype(stride<2>(tCgCol))::value == 0, "Expected stride-0 CPY_N for col broadcast"); |
| 401 | static_assert(decltype(stride<4>(tCgCol))::value == 0, "Expected stride-0 EPI_N for col broadcast"); |
| 402 | |
| 403 | auto tCgCol_s = tCgCol(_,_,0,_,0); // (CPY,CPY_M,EPI_M) |
| 404 | auto tCrCol_s = tCrCol(_,_,0,_,0); // (CPY,CPY_M,EPI_M) |
| 405 | auto tCcCol_s = tCcCol(_,_,0,_,0); // (CPY,CPY_M,EPI_M) |
| 406 | |
| 407 | cute::Tensor pred = make_tensor<bool>(shape(tCgCol_s)); |
| 408 | CUTLASS_PRAGMA_UNROLL |
| 409 | for (int i = 0; i < size(pred); ++i) { |
| 410 | pred(i) = get<0>(tCcCol_s(i)) < m; |
| 411 | } |
| 412 | |
| 413 | copy_if(pred, tCgCol_s, tCrCol_s); |
| 414 | } |
| 415 | |
| 416 | template <typename ElementAccumulator, int FragmentSize> |
| 417 | CUTLASS_DEVICE Array<Element, FragmentSize> |
| 418 | visit(Array<ElementAccumulator, FragmentSize> const& frg_acc, int epi_v, int epi_m, int epi_n) { |
| 419 | Array<Element, FragmentSize> frg_col; |
| 420 | cute::Tensor tCrCol_mn = tCrCol(_,_,_,epi_m,epi_n); |
| 421 | |
| 422 | CUTLASS_PRAGMA_UNROLL |
| 423 | for (int i = 0; i < FragmentSize; ++i) { |
| 424 | frg_col[i] = tCrCol_mn(epi_v * FragmentSize + i); |
| 425 | } |
| 426 | |
| 427 | return frg_col; |
| 428 | } |
| 429 | |
| 430 | }; |
| 431 | |
| 432 | template < |
| 433 | bool ReferenceSrc, // do register tensors reference the src or dst layout of the tiled copy |
| 434 | class... Args |
| 435 | > |
| 436 | CUTLASS_DEVICE auto |
| 437 | get_consumer_store_callbacks(ConsumerStoreArgs<Args...> const& args) { |
| 438 | |
| 439 | auto [M, N, K, L] = args.problem_shape_mnkl; |
| 440 | auto [m, n, k, l] = args.tile_coord_mnkl; |
| 441 | |
| 442 | cute::Tensor mCol = make_tensor(make_gmem_ptr(params.ptr_col_array[l]), make_shape(M,N,1), params.dCol); |
| 443 | cute::Tensor tCgCol = sm90_partition_for_epilogue<ReferenceSrc>( // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) |
| 444 | mCol, args.tile_shape_mnk, args.tile_coord_mnkl, args.epi_tile, args.tiled_copy, args.thread_idx); |
| 445 | cute::Tensor tCrCol = make_tensor_like(tCgCol); // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) |
| 446 | |
| 447 | // Generate an identity tensor matching the shape of the global tensor and |
| 448 | // partition the same way, this will be used to generate the predicate |
| 449 | // tensor for loading |
| 450 | cute::Tensor cCol = make_identity_tensor(mCol.shape()); |
| 451 | cute::Tensor tCcCol = sm90_partition_for_epilogue<ReferenceSrc>( // (CPY,CPY_M,CPY_N,EPI_M,EPI_N) |
| 452 | cCol, args.tile_shape_mnk, args.tile_coord_mnkl, args.epi_tile, args.tiled_copy, args.thread_idx); |
| 453 | |
| 454 | return ConsumerStoreCallbacks( |
| 455 | cute::move(tCgCol), |
| 456 | cute::move(tCrCol), |
| 457 | cute::move(tCcCol), |
| 458 | args.problem_shape_mnkl, |
| 459 | l, |
| 460 | params |
| 461 | ); |
| 462 | } |
| 463 | }; |
| 464 | |
| 465 | } |