A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Kernel] Add punica dimensions for Granite 13b (#5559)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Jun 17, 2024 at 21:54 UTC
5002175e801703c5b8a1411b490f6ff6c1747c8e
2 files changed
+12
csrc/punica/bgmv/bgmv_config.h
+8
@@ -20,6 +20,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
20
f(in_T, out_T, W_T, narrow, 1152) \
21
f(in_T, out_T, W_T, narrow, 1280) \
22
f(in_T, out_T, W_T, narrow, 1536) \
23
+ f(in_T, out_T, W_T, narrow, 1664) \
24
f(in_T, out_T, W_T, narrow, 1728) \
25
f(in_T, out_T, W_T, narrow, 1792) \
26
f(in_T, out_T, W_T, narrow, 2048) \
@@ -36,6 +37,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
37
f(in_T, out_T, W_T, narrow, 5120) \
38
f(in_T, out_T, W_T, narrow, 5504) \
39
f(in_T, out_T, W_T, narrow, 5632) \
40
+ f(in_T, out_T, W_T, narrow, 5888) \
41
f(in_T, out_T, W_T, narrow, 6144) \
42
f(in_T, out_T, W_T, narrow, 6400) \
43
f(in_T, out_T, W_T, narrow, 6848) \
@@ -45,6 +47,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
47
f(in_T, out_T, W_T, narrow, 9216) \
48
f(in_T, out_T, W_T, narrow, 10240) \
49
f(in_T, out_T, W_T, narrow, 11008) \
50
+ f(in_T, out_T, W_T, narrow, 11264) \
51
f(in_T, out_T, W_T, narrow, 12288) \
52
f(in_T, out_T, W_T, narrow, 13696) \
53
f(in_T, out_T, W_T, narrow, 13824) \
@@ -53,6 +56,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
56
f(in_T, out_T, W_T, narrow, 16384) \
57
f(in_T, out_T, W_T, narrow, 20480) \
58
f(in_T, out_T, W_T, narrow, 22016) \
59
+ f(in_T, out_T, W_T, narrow, 22528) \
60
f(in_T, out_T, W_T, narrow, 24576) \
61
f(in_T, out_T, W_T, narrow, 27392) \
62
f(in_T, out_T, W_T, narrow, 27648) \
@@ -91,6 +95,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
95
f(in_T, out_T, W_T, 1152, narrow) \
96
f(in_T, out_T, W_T, 1280, narrow) \
97
f(in_T, out_T, W_T, 1536, narrow) \
98
+ f(in_T, out_T, W_T, 1664, narrow) \
99
f(in_T, out_T, W_T, 1728, narrow) \
100
f(in_T, out_T, W_T, 1792, narrow) \
101
f(in_T, out_T, W_T, 2048, narrow) \
@@ -107,6 +112,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
112
f(in_T, out_T, W_T, 5120, narrow) \
113
f(in_T, out_T, W_T, 5504, narrow) \
114
f(in_T, out_T, W_T, 5632, narrow) \
115
+ f(in_T, out_T, W_T, 5888, narrow) \
116
f(in_T, out_T, W_T, 6144, narrow) \
117
f(in_T, out_T, W_T, 6400, narrow) \
118
f(in_T, out_T, W_T, 6848, narrow) \
@@ -116,6 +122,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
122
f(in_T, out_T, W_T, 9216, narrow) \
123
f(in_T, out_T, W_T, 10240, narrow) \
124
f(in_T, out_T, W_T, 11008, narrow) \
125
+ f(in_T, out_T, W_T, 11264, narrow) \
126
f(in_T, out_T, W_T, 12288, narrow) \
127
f(in_T, out_T, W_T, 13696, narrow) \
128
f(in_T, out_T, W_T, 13824, narrow) \
@@ -124,6 +131,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
131
f(in_T, out_T, W_T, 16384, narrow) \
132
f(in_T, out_T, W_T, 20480, narrow) \
133
f(in_T, out_T, W_T, 22016, narrow) \
134
+ f(in_T, out_T, W_T, 22528, narrow) \
135
f(in_T, out_T, W_T, 24576, narrow) \
136
f(in_T, out_T, W_T, 27392, narrow) \
137
f(in_T, out_T, W_T, 27648, narrow) \
tests/lora/test_punica.py
+4
@@ -53,6 +53,7 @@ H1 = H2 = [
53
1152,
54
1280,
55
1536,
56
+ 1664,
57
2048,
58
2304,
59
2560,
@@ -66,6 +67,7 @@ H1 = H2 = [
67
5120,
68
5504,
69
5632,
70
+ 5888,
71
6144,
72
6400,
73
6848,
@@ -75,10 +77,12 @@ H1 = H2 = [
77
9216,
78
10240,
79
11008,
80
+ 11264,
81
13824,
82
14336,
83
15360,
84
22016,
85
+ 22528,
86
24576,
87
27392,
88
27648,