A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] Marlin 2:4 temp fix for large M dim (>256) (#10464)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Lucas Wilkinson committed
Nov 19, 2024 at 22:40 UTC
d200972e7f4969da50f533b46c856c5ff5a9d27d
2 files changed
+13
-4
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+11
-4
@@ -910,13 +910,16 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
910
// than better compute utilization
911
thread_k = 128;
912
thread_m = 128;
913
- } else if (prob_n <= 256) {
913
+ } else {
914
thread_k = 64;
915
thread_m = 256;
916
- } else {
917
- thread_k = 32;
918
- thread_m = 512;
916
}
917
+ // Also had
918
+ // if prob_n > 256
919
+ // thread_k = 32;
920
+ // thread_m = 512;
921
+ // but this is broken,
922
+ // TODO(Lucas, Alex M): figure out why
923
}
924
925
int thread_k_blocks = thread_k / 32; // 2:4 version with m16n8k32 instruction
@@ -1079,6 +1082,8 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1082
// Verify A device and strides
1083
TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
1084
TORCH_CHECK(a.is_contiguous(), "A is not contiguous");
1085
+ TORCH_CHECK(a.dtype() == torch::kFloat16,
1086
+ "A is not float16, currently only float16 is supported");
1087
1088
// Verify B device and strides
1089
TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
@@ -1091,6 +1096,8 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1096
// Verify scales device and strides
1097
TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU");
1098
TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous");
1099
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat16,
1100
+ "A is not float16, currently only float16 is supported");
1101
1102
// Alloc C matrix
1103
const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
tests/kernels/test_marlin_gemm.py
+2
@@ -50,6 +50,8 @@ MNK_FACTORS = [
50
(13, 17, 67),
51
(26, 37, 13),
52
(67, 13, 11),
53
+ (257, 13, 11),
54
+ (658, 13, 11),
55
]
56
57
DTYPES = [torch.float16, torch.bfloat16]