A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Kernel] Add punica dimension for Swallow-MS-7B LoRA (#4134)
Shoichi Uchinami committed
Apr 18, 2024 at 02:02 UTC
a53222544c6385ee314a26fdf42eb14f5b4e5ad9
2 files changed
+2
csrc/punica/bgmv/bgmv_config.h
+1
@@ -60,6 +60,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
60
f(in_T, out_T, W_T, narrow, 32768) \
61
f(in_T, out_T, W_T, narrow, 33024) \
62
f(in_T, out_T, W_T, narrow, 36864) \
63
+ f(in_T, out_T, W_T, narrow, 43264) \
64
f(in_T, out_T, W_T, narrow, 49152) \
65
f(in_T, out_T, W_T, narrow, 64000) \
66
f(in_T, out_T, W_T, narrow, 64256) \
tests/lora/test_punica.py
+1
@@ -82,6 +82,7 @@ H1 = H2 = [
82
32768,
83
33024,
84
36864,
85
+ 43264,
86
49152,
87
64000,
88
64256,