@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 55712941e5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bug Fix] Illegal memory access, FP8 Llama 3.1 405b (#6852)

Lucas Wilkinson committed Jul 26, 2024 at 22:27 UTC 55712941e57bcfd662db2905811d6e2807b9153f
1 file changed +37 -9
csrc/quantization/cutlass_w8a8/broadcast_load_epilogue_c3x.hpp
+37 -9
@@ -328,20 +328,36 @@ struct Sm90ColOrScalarBroadcast {
328 return EmptyProducerLoadCallbacks{};
329 }
330
331 - template<class GTensor, class RTensor>
331 + template<class GTensor, class RTensor, class CTensor, class ProblemShape>
332 struct ConsumerStoreCallbacks : EmptyConsumerStoreCallbacks {
333 CUTLASS_DEVICE
334 - ConsumerStoreCallbacks(GTensor&& tCgCol, RTensor&& tCrCol, Params const& params)
335 - : tCgCol(cute::forward<GTensor>(tCgCol)),
336 - tCrCol(cute::forward<RTensor>(tCrCol)),
337 - params(params) {}
334 + ConsumerStoreCallbacks(
335 + GTensor&& tCgCol,
336 + RTensor&& tCrCol,
337 + CTensor&& tCcCol,
338 + ProblemShape problem_shape,
339 + Params const& params
340 + ):
341 + tCgCol(cute::forward<GTensor>(tCgCol)),
342 + tCrCol(cute::forward<RTensor>(tCrCol)),
343 + tCcCol(cute::forward<CTensor>(tCcCol)),
344 + m(get<0>(problem_shape)),
345 + params(params) {}
346
347 GTensor tCgCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
340 - RTensor tCrCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
348 + RTensor tCrCol;
349 + CTensor tCcCol; // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
350 Params const& params;
351 + int m;
352
353 CUTLASS_DEVICE void
354 begin() {
355 + Tensor pred = make_tensor<bool>(shape(tCgCol));
356 + CUTLASS_PRAGMA_UNROLL
357 + for (int i = 0; i < size(pred); ++i) {
358 + pred(i) = get<0>(tCcCol(i)) < m;
359 + }
360 +
361 if (!params.col_broadcast) {
362 fill(tCrCol, *(params.ptr_col));
363 return;
@@ -349,7 +365,7 @@ struct Sm90ColOrScalarBroadcast {
365
366 // Filter so we don't issue redundant copies over stride-0 modes
367 // (only works if 0-strides are in same location, which is by construction)
352 - copy_aligned(filter(tCgCol), filter(tCrCol));
368 + copy_if(pred, filter(tCgCol), filter(tCrCol));
369 }
370
371 template <typename ElementAccumulator, int FragmentSize>
@@ -381,8 +397,20 @@ struct Sm90ColOrScalarBroadcast {
397 mCol, args.tile_shape_mnk, args.tile_coord_mnkl, args.epi_tile, args.tiled_copy, args.thread_idx);
398 Tensor tCrCol = make_tensor_like(tCgCol); // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
399
384 - return ConsumerStoreCallbacks<decltype(tCgCol), decltype(tCrCol)>(
385 - cute::move(tCgCol), cute::move(tCrCol), params);
400 + // Generate an identity tensor matching the shape of the global tensor and
401 + // partition the same way, this will be used to generate the predicate
402 + // tensor for loading
403 + Tensor cCol = make_identity_tensor(mCol.shape());
404 + Tensor tCcCol = sm90_partition_for_epilogue<ReferenceSrc>( // (CPY,CPY_M,CPY_N,EPI_M,EPI_N)
405 + cCol, args.tile_shape_mnk, args.tile_coord_mnkl, args.epi_tile, args.tiled_copy, args.thread_idx);
406 +
407 + return ConsumerStoreCallbacks(
408 + cute::move(tCgCol),
409 + cute::move(tCrCol),
410 + cute::move(tCcCol),
411 + args.problem_shape_mnkl,
412 + params
413 + );
414 }
415 };
416