A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Kernel] Add punica dimensions for Granite 3b and 8b (#5930)
Signed-off-by: Joe Runde <joe@joerun.de>
Joe Runde committed
Jun 28, 2024 at 20:48 UTC
ba4994443afc6a8249ed726c5ebd09b2c57a3b00
2 files changed
+3
csrc/punica/bgmv/bgmv_config.h
+2
@@ -86,6 +86,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
86
f(in_T, out_T, W_T, narrow, 36864) \
87
f(in_T, out_T, W_T, narrow, 43264) \
88
f(in_T, out_T, W_T, narrow, 49152) \
89
+ f(in_T, out_T, W_T, narrow, 49408) \
90
f(in_T, out_T, W_T, narrow, 60544) \
91
f(in_T, out_T, W_T, narrow, 60672) \
92
f(in_T, out_T, W_T, narrow, 64000) \
@@ -182,6 +183,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
183
f(in_T, out_T, W_T, 36864, narrow) \
184
f(in_T, out_T, W_T, 43264, narrow) \
185
f(in_T, out_T, W_T, 49152, narrow) \
186
+ f(in_T, out_T, W_T, 49408, narrow) \
187
f(in_T, out_T, W_T, 60544, narrow) \
188
f(in_T, out_T, W_T, 60672, narrow) \
189
f(in_T, out_T, W_T, 64000, narrow) \
tests/lora/test_punica.py
+1
@@ -111,6 +111,7 @@ H1 = H2 = [
111
36864,
112
43264,
113
49152,
114
+ 49408,
115
60544,
116
60672,
117
64000,