A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Kernel] Add punica dimension for Qwen1.5-32B LoRA (#4850)
Co-authored-by: Silencio <silencio@adsl-99-6-187-6.dsl.irvnca.sbcglobal.net>
Silencio committed
May 17, 2024 at 02:16 UTC
8435b207af398cb6cec961f8ac8e1d8bb5164b3e
2 files changed
+3
csrc/punica/bgmv/bgmv_config.h
+2
@@ -53,6 +53,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
53
f(in_T, out_T, W_T, narrow, 22016) \
54
f(in_T, out_T, W_T, narrow, 24576) \
55
f(in_T, out_T, W_T, narrow, 27392) \
56
+ f(in_T, out_T, W_T, narrow, 27648) \
57
f(in_T, out_T, W_T, narrow, 28672) \
58
f(in_T, out_T, W_T, narrow, 32000) \
59
f(in_T, out_T, W_T, narrow, 32256) \
@@ -121,6 +122,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
122
f(in_T, out_T, W_T, 22016, narrow) \
123
f(in_T, out_T, W_T, 24576, narrow) \
124
f(in_T, out_T, W_T, 27392, narrow) \
125
+ f(in_T, out_T, W_T, 27648, narrow) \
126
f(in_T, out_T, W_T, 28672, narrow) \
127
f(in_T, out_T, W_T, 32000, narrow) \
128
f(in_T, out_T, W_T, 32256, narrow) \
tests/lora/test_punica.py
+1
@@ -79,6 +79,7 @@ H1 = H2 = [
79
22016,
80
24576,
81
27392,
82
+ 27648,
83
32000,
84
32256,
85
32512,