A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bug Fix] Illegal memory access, FP8 Llama 3.1 405b (#6852)
Lucas Wilkinson committed
Jul 26, 2024 at 22:27 UTC
55712941e57bcfd662db2905811d6e2807b9153f
1 file changed
+37
-9
csrc/quantization/cutlass_w8a8/broadcast_load_epilogue_c3x.hpp
+37
-9
@@ -328,20 +328,36 @@ struct Sm90ColOrScalarBroadcast {
328
return EmptyProducerLoadCallbacks{};
329
}
330
331
- template<class GTensor, class RTensor>
331
+ template<class GTensor, class RTensor, class CTensor, class ProblemShape>
332
struct ConsumerStoreCallbacks : EmptyConsumerStoreCallbacks {
333
CUTLASS_DEVICE
334
- ConsumerStoreCallbacks(GTensor&& tCgCol, RTensor&& tCrCol, Params const& params)
335
- : tCgCol(cute::forward<GTensor>(tCgCol)),
336
- tCrCol(cute::forward<RTensor>(tCrCol)),
337
- params(params) {}
334
+ ConsumerStoreCallbacks(
335
+ GTensor&& tCgCol,
336
+ RTensor&& tCrCol,
337
+ CTensor&& tCcCol,
338
+ ProblemShape problem_shape,
339
+ Params const& params
340
+ ):
341
+ tCgCol(cute::forward<GTensor>(tCgCol)),
342
+ tCrCol(cute::forward<RTensor>(tCrCol)),
343
+ tCcCol(cute::forward<CTensor>(tCcCol)),
344
+ m(get<0>(problem_shape)),
345
+ params(params) {}
346
347
GTensor tCgCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
340
- RTensor tCrCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
348
+ RTensor tCrCol;
349
+ CTensor tCcCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
350
Params const& params;
351
+ int m;
352
353
CUTLASS_DEVICE void
354
begin() {
355
+ Tensor pred = make_tensor<bool>(shape(tCgCol));
356
+ CUTLASS_PRAGMA_UNROLL
357
+ for (int i = 0; i < size(pred); ++i) {
358
+ pred(i) = get<0>(tCcCol(i)) < m;
359
+ }
360
+
361
if (!params.col_broadcast) {
362
fill(tCrCol, *(params.ptr_col));
363
return;
@@ -349,7 +365,7 @@ struct Sm90ColOrScalarBroadcast {
365
366
// Filter so we don't issue redundant copies over stride-0 modes
367
// (only works if 0-strides are in same location, which is by construction)
352
- copy_aligned(filter(tCgCol), filter(tCrCol));
368
+ copy_if(pred, filter(tCgCol), filter(tCrCol));
369
}
370
371
template <typename ElementAccumulator, int FragmentSize>
@@ -381,8 +397,20 @@ struct Sm90ColOrScalarBroadcast {
397
mCol, args.tile_shape_mnk, args.tile_coord_mnkl, args.epi_tile, args.tiled_copy, args.thread_idx);
398
Tensor tCrCol = make_tensor_like(tCgCol); // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
399
384
- return ConsumerStoreCallbacks<decltype(tCgCol), decltype(tCrCol)>(
385
- cute::move(tCgCol), cute::move(tCrCol), params);
400
+ // Generate an identity tensor matching the shape of the global tensor and
401
+ // partition the same way, this will be used to generate the predicate
402
+ // tensor for loading
403
+ Tensor cCol = make_identity_tensor(mCol.shape());
404
+ Tensor tCcCol = sm90_partition_for_epilogue<ReferenceSrc>( // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
405
+ cCol, args.tile_shape_mnk, args.tile_coord_mnkl, args.epi_tile, args.tiled_copy, args.thread_idx);
406
+
407
+ return ConsumerStoreCallbacks(
408
+ cute::move(tCgCol),
409
+ cute::move(tCrCol),
410
+ cute::move(tCcCol),
411
+ args.problem_shape_mnkl,
412
+ params
413
+ );
414
}
415
};
416