@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 781e3b9a42
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix][Kernel] Fix build for sm_60 in GGUF kernel (#8506)

sasha0552 committed Sep 16, 2024 at 18:15 UTC 781e3b9a4281babf5576f8dc22445af69814d8f6
1 file changed +4
csrc/quantization/gguf/vecdotq.cuh
+4
@@ -1671,6 +1671,7 @@ static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(
1671
1672 static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(
1673 const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1674 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1675 const block_iq1_s * bq1 = (const block_iq1_s *) vbq;
1676
1677 const int qs_packed = get_int_b2(bq1->qs, iqs);
@@ -1697,10 +1698,12 @@ static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(
1698 const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);
1699 const float2 ds = __half22float2(bq8_1[iqs].ds);
1700 return d1q * (ds.x*sumi + ds.y*delta);
1701 +#endif
1702 }
1703
1704 static __device__ __forceinline__ float vec_dot_iq1_m_q8_1(
1705 const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) {
1706 +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610
1707
1708 const block_iq1_m * bq1 = (const block_iq1_m *) vbq;
1709
@@ -1741,6 +1744,7 @@ static __device__ __forceinline__ float vec_dot_iq1_m_q8_1(
1744 const int sc0 = 2*((tmp >> 0) & 0x07) + 1;
1745 const int sc1 = 2*((tmp >> 3) & 0x07) + 1;
1746 return d * ((sumi[0] + sumf[0]) * sc0 + (sumi[1] + sumf[1]) * sc1);
1747 +#endif
1748 }
1749
1750 static __device__ __forceinline__ void get_int_from_table_16(const uint32_t & q4, const uint8_t * values,