@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d200972e7f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Marlin 2:4 temp fix for large M dim (>256) (#10464)

Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>

Lucas Wilkinson committed Nov 19, 2024 at 22:40 UTC d200972e7f4969da50f533b46c856c5ff5a9d27d
2 files changed +13 -4
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+11 -4
@@ -910,13 +910,16 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
910 // than better compute utilization
911 thread_k = 128;
912 thread_m = 128;
913 - } else if (prob_n <= 256) {
913 + } else {
914 thread_k = 64;
915 thread_m = 256;
916 - } else {
917 - thread_k = 32;
918 - thread_m = 512;
916 }
917 + // Also had
918 + // if prob_n > 256
919 + // thread_k = 32;
920 + // thread_m = 512;
921 + // but this is broken,
922 + // TODO(Lucas, Alex M): figure out why
923 }
924
925 int thread_k_blocks = thread_k / 32; // 2:4 version with m16n8k32 instruction
@@ -1079,6 +1082,8 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1082 // Verify A device and strides
1083 TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
1084 TORCH_CHECK(a.is_contiguous(), "A is not contiguous");
1085 + TORCH_CHECK(a.dtype() == torch::kFloat16,
1086 + "A is not float16, currently only float16 is supported");
1087
1088 // Verify B device and strides
1089 TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
@@ -1091,6 +1096,8 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1096 // Verify scales device and strides
1097 TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU");
1098 TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous");
1099 + TORCH_CHECK(b_scales.dtype() == torch::kFloat16,
1100 + "A is not float16, currently only float16 is supported");
1101
1102 // Alloc C matrix
1103 const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
tests/kernels/test_marlin_gemm.py
+2
@@ -50,6 +50,8 @@ MNK_FACTORS = [
50 (13, 17, 67),
51 (26, 37, 13),
52 (67, 13, 11),
53 + (257, 13, 11),
54 + (658, 13, 11),
55 ]
56
57 DTYPES = [torch.float16, torch.bfloat16]