A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Machete garbage results for some models (large K dim) (#9212)
Lucas Wilkinson committed
Oct 10, 2024 at 02:16 UTC
a64e7b940734b68d849ed2b07ca1bc3824713555
2 files changed
+16
-12
csrc/quantization/machete/machete_mainloop.cuh
+13
-10
@@ -591,24 +591,27 @@ struct MacheteCollectiveMma {
591
tma_load_b = make_tma_copy_B(
592
make_logical_tensor(ptr_B, make_shape(N, K, L), args.dB));
593
594
+ int32_t scale_k =
595
+ (ModeHasScales) ? (K + args.group_size - 1) / args.group_size : 0;
596
+ int32_t group_size = (ModeHasScales) ? args.group_size : 0;
597
+
598
if constexpr (ModeHasScales) {
595
- tma_load_scale = make_tma_copy_scale(make_logical_tensor(
596
- args.ptr_S, make_shape(M, args.group_size, L), args.dS));
599
+ tma_load_scale = make_tma_copy_scale(
600
+ make_logical_tensor(args.ptr_S, make_shape(M, scale_k, L), args.dS));
601
}
602
603
if constexpr (KernelConversionMode ==
604
ConversionMode::ConvertAndScaleWithZero) {
601
- tma_load_zero = make_tma_copy_zero(make_logical_tensor(
602
- args.ptr_Z, make_shape(M, args.group_size, L), args.dS));
605
+ tma_load_zero = make_tma_copy_zero(
606
+ make_logical_tensor(args.ptr_Z, make_shape(M, scale_k, L), args.dS));
607
}
608
605
- if constexpr (KernelConversionMode == ConversionMode::DirectConvert) {
606
- return {tma_load_a, tma_load_b, tma_load_scale, tma_load_zero, 0, 0};
607
- } else if constexpr (ModeHasScales) {
608
- auto scale_k = (K + args.group_size - 1) / args.group_size;
609
-
609
+ if constexpr (KernelConversionMode == ConversionMode::DirectConvert ||
610
+ KernelConversionMode == ConversionMode::ConvertAndScale ||
611
+ KernelConversionMode ==
612
+ ConversionMode::ConvertAndScaleWithZero) {
613
return {tma_load_a, tma_load_b, tma_load_scale,
611
- tma_load_zero, scale_k, args.group_size};
614
+ tma_load_zero, scale_k, group_size};
615
} else {
616
static_assert(cutlass::detail::dependent_false<KernelSchedule>,
617
"Conversion mode not handled in to_underlying_arguments.");
tests/kernels/test_machete_gemm.py
+3
-2
@@ -24,13 +24,14 @@ MNK_SHAPES = [
24
(1, 128, 128),
25
(1, 512, 1024),
26
(1, 4096, 4096),
27
+ (1, 8192, 28672),
28
(13, 8192, 4096),
29
(26, 4096, 8192),
29
- (1, 4096, 4096),
30
+ (64, 4096, 4096),
31
+ (64, 8192, 28672),
32
(257, 128, 4096),
33
(257, 4224, 4160),
34
(257, 4096, 4096),
33
- (64, 4096, 4096),
35
(1024, 4096, 8192),
36
(1024, 8192, 4096),
37
]