@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ba4994443a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Kernel] Add punica dimensions for Granite 3b and 8b (#5930)

Signed-off-by: Joe Runde <joe@joerun.de>

Joe Runde committed Jun 28, 2024 at 20:48 UTC ba4994443afc6a8249ed726c5ebd09b2c57a3b00
2 files changed +3
csrc/punica/bgmv/bgmv_config.h
+2
@@ -86,6 +86,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
86 f(in_T, out_T, W_T, narrow, 36864) \
87 f(in_T, out_T, W_T, narrow, 43264) \
88 f(in_T, out_T, W_T, narrow, 49152) \
89 + f(in_T, out_T, W_T, narrow, 49408) \
90 f(in_T, out_T, W_T, narrow, 60544) \
91 f(in_T, out_T, W_T, narrow, 60672) \
92 f(in_T, out_T, W_T, narrow, 64000) \
@@ -182,6 +183,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
183 f(in_T, out_T, W_T, 36864, narrow) \
184 f(in_T, out_T, W_T, 43264, narrow) \
185 f(in_T, out_T, W_T, 49152, narrow) \
186 + f(in_T, out_T, W_T, 49408, narrow) \
187 f(in_T, out_T, W_T, 60544, narrow) \
188 f(in_T, out_T, W_T, 60672, narrow) \
189 f(in_T, out_T, W_T, 64000, narrow) \
tests/lora/test_punica.py
+1
@@ -111,6 +111,7 @@ H1 = H2 = [
111 36864,
112 43264,
113 49152,
114 + 49408,
115 60544,
116 60672,
117 64000,