@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a64e7b9407
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Machete garbage results for some models (large K dim) (#9212)

Lucas Wilkinson committed Oct 10, 2024 at 02:16 UTC a64e7b940734b68d849ed2b07ca1bc3824713555
2 files changed +16 -12
csrc/quantization/machete/machete_mainloop.cuh
+13 -10
@@ -591,24 +591,27 @@ struct MacheteCollectiveMma {
591 tma_load_b = make_tma_copy_B(
592 make_logical_tensor(ptr_B, make_shape(N, K, L), args.dB));
593
594 + int32_t scale_k =
595 + (ModeHasScales) ? (K + args.group_size - 1) / args.group_size : 0;
596 + int32_t group_size = (ModeHasScales) ? args.group_size : 0;
597 +
598 if constexpr (ModeHasScales) {
595 - tma_load_scale = make_tma_copy_scale(make_logical_tensor(
596 - args.ptr_S, make_shape(M, args.group_size, L), args.dS));
599 + tma_load_scale = make_tma_copy_scale(
600 + make_logical_tensor(args.ptr_S, make_shape(M, scale_k, L), args.dS));
601 }
602
603 if constexpr (KernelConversionMode ==
604 ConversionMode::ConvertAndScaleWithZero) {
601 - tma_load_zero = make_tma_copy_zero(make_logical_tensor(
602 - args.ptr_Z, make_shape(M, args.group_size, L), args.dS));
605 + tma_load_zero = make_tma_copy_zero(
606 + make_logical_tensor(args.ptr_Z, make_shape(M, scale_k, L), args.dS));
607 }
608
605 - if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
606 - return {tma_load_a, tma_load_b, tma_load_scale, tma_load_zero, 0, 0};
607 - } else if constexpr (ModeHasScales) {
608 - auto scale_k = (K + args.group_size - 1) / args.group_size;
609 -
609 + if constexpr (KernelConversionMode == ConversionMode::DirectConvert ||
610 + KernelConversionMode == ConversionMode::ConvertAndScale ||
611 + KernelConversionMode ==
612 + ConversionMode::ConvertAndScaleWithZero) {
613 return {tma_load_a, tma_load_b, tma_load_scale,
611 - tma_load_zero, scale_k, args.group_size};
614 + tma_load_zero, scale_k, group_size};
615 } else {
616 static_assert(cutlass::detail::dependent_false<KernelSchedule>,
617 "Conversion mode not handled in to_underlying_arguments.");
tests/kernels/test_machete_gemm.py
+3 -2
@@ -24,13 +24,14 @@ MNK_SHAPES = [
24 (1, 128, 128),
25 (1, 512, 1024),
26 (1, 4096, 4096),
27 + (1, 8192, 28672),
28 (13, 8192, 4096),
29 (26, 4096, 8192),
29 - (1, 4096, 4096),
30 + (64, 4096, 4096),
31 + (64, 8192, 28672),
32 (257, 128, 4096),
33 (257, 4224, 4160),
34 (257, 4096, 4096),
33 - (64, 4096, 4096),
35 (1024, 4096, 8192),
36 (1024, 8192, 4096),
37 ]