A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
Missing comment explaining VDR variable in GGUF kernels (#13290)
Szymon Ożóg committed
Feb 21, 2025 at 07:06 UTC
1cdc88614a1cebd7b277be7a19fd7f36746ac269
1 file changed
+2
csrc/quantization/gguf/vecdotq.cuh
+2
@@ -37,6 +37,8 @@ static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t *
37
return *((const int *) (x8 + sizeof(int) * i32)); // assume at least 4 byte alignment
38
}
39
40
+// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called
41
+// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q
42
43
#define VDR_Q4_0_Q8_1_MMVQ 2
44
#define VDR_Q4_0_Q8_1_MMQ 4