@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5002175e80
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Kernel] Add punica dimensions for Granite 13b (#5559)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Jun 17, 2024 at 21:54 UTC 5002175e801703c5b8a1411b490f6ff6c1747c8e
2 files changed +12
csrc/punica/bgmv/bgmv_config.h
+8
@@ -20,6 +20,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
20 f(in_T, out_T, W_T, narrow, 1152) \
21 f(in_T, out_T, W_T, narrow, 1280) \
22 f(in_T, out_T, W_T, narrow, 1536) \
23 + f(in_T, out_T, W_T, narrow, 1664) \
24 f(in_T, out_T, W_T, narrow, 1728) \
25 f(in_T, out_T, W_T, narrow, 1792) \
26 f(in_T, out_T, W_T, narrow, 2048) \
@@ -36,6 +37,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
37 f(in_T, out_T, W_T, narrow, 5120) \
38 f(in_T, out_T, W_T, narrow, 5504) \
39 f(in_T, out_T, W_T, narrow, 5632) \
40 + f(in_T, out_T, W_T, narrow, 5888) \
41 f(in_T, out_T, W_T, narrow, 6144) \
42 f(in_T, out_T, W_T, narrow, 6400) \
43 f(in_T, out_T, W_T, narrow, 6848) \
@@ -45,6 +47,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
47 f(in_T, out_T, W_T, narrow, 9216) \
48 f(in_T, out_T, W_T, narrow, 10240) \
49 f(in_T, out_T, W_T, narrow, 11008) \
50 + f(in_T, out_T, W_T, narrow, 11264) \
51 f(in_T, out_T, W_T, narrow, 12288) \
52 f(in_T, out_T, W_T, narrow, 13696) \
53 f(in_T, out_T, W_T, narrow, 13824) \
@@ -53,6 +56,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
56 f(in_T, out_T, W_T, narrow, 16384) \
57 f(in_T, out_T, W_T, narrow, 20480) \
58 f(in_T, out_T, W_T, narrow, 22016) \
59 + f(in_T, out_T, W_T, narrow, 22528) \
60 f(in_T, out_T, W_T, narrow, 24576) \
61 f(in_T, out_T, W_T, narrow, 27392) \
62 f(in_T, out_T, W_T, narrow, 27648) \
@@ -91,6 +95,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
95 f(in_T, out_T, W_T, 1152, narrow) \
96 f(in_T, out_T, W_T, 1280, narrow) \
97 f(in_T, out_T, W_T, 1536, narrow) \
98 + f(in_T, out_T, W_T, 1664, narrow) \
99 f(in_T, out_T, W_T, 1728, narrow) \
100 f(in_T, out_T, W_T, 1792, narrow) \
101 f(in_T, out_T, W_T, 2048, narrow) \
@@ -107,6 +112,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
112 f(in_T, out_T, W_T, 5120, narrow) \
113 f(in_T, out_T, W_T, 5504, narrow) \
114 f(in_T, out_T, W_T, 5632, narrow) \
115 + f(in_T, out_T, W_T, 5888, narrow) \
116 f(in_T, out_T, W_T, 6144, narrow) \
117 f(in_T, out_T, W_T, 6400, narrow) \
118 f(in_T, out_T, W_T, 6848, narrow) \
@@ -116,6 +122,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
122 f(in_T, out_T, W_T, 9216, narrow) \
123 f(in_T, out_T, W_T, 10240, narrow) \
124 f(in_T, out_T, W_T, 11008, narrow) \
125 + f(in_T, out_T, W_T, 11264, narrow) \
126 f(in_T, out_T, W_T, 12288, narrow) \
127 f(in_T, out_T, W_T, 13696, narrow) \
128 f(in_T, out_T, W_T, 13824, narrow) \
@@ -124,6 +131,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
131 f(in_T, out_T, W_T, 16384, narrow) \
132 f(in_T, out_T, W_T, 20480, narrow) \
133 f(in_T, out_T, W_T, 22016, narrow) \
134 + f(in_T, out_T, W_T, 22528, narrow) \
135 f(in_T, out_T, W_T, 24576, narrow) \
136 f(in_T, out_T, W_T, 27392, narrow) \
137 f(in_T, out_T, W_T, 27648, narrow) \
tests/lora/test_punica.py
+4
@@ -53,6 +53,7 @@ H1 = H2 = [
53 1152,
54 1280,
55 1536,
56 + 1664,
57 2048,
58 2304,
59 2560,
@@ -66,6 +67,7 @@ H1 = H2 = [
67 5120,
68 5504,
69 5632,
70 + 5888,
71 6144,
72 6400,
73 6848,
@@ -75,10 +77,12 @@ H1 = H2 = [
77 9216,
78 10240,
79 11008,
80 + 11264,
81 13824,
82 14336,
83 15360,
84 22016,
85 + 22528,
86 24576,
87 27392,
88 27648,