@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8435b207af
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Kernel] Add punica dimension for Qwen1.5-32B LoRA (#4850)

Co-authored-by: Silencio <silencio@adsl-99-6-187-6.dsl.irvnca.sbcglobal.net>

Silencio committed May 17, 2024 at 02:16 UTC 8435b207af398cb6cec961f8ac8e1d8bb5164b3e
2 files changed +3
csrc/punica/bgmv/bgmv_config.h
+2
@@ -53,6 +53,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
53 f(in_T, out_T, W_T, narrow, 22016) \
54 f(in_T, out_T, W_T, narrow, 24576) \
55 f(in_T, out_T, W_T, narrow, 27392) \
56 + f(in_T, out_T, W_T, narrow, 27648) \
57 f(in_T, out_T, W_T, narrow, 28672) \
58 f(in_T, out_T, W_T, narrow, 32000) \
59 f(in_T, out_T, W_T, narrow, 32256) \
@@ -121,6 +122,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
122 f(in_T, out_T, W_T, 22016, narrow) \
123 f(in_T, out_T, W_T, 24576, narrow) \
124 f(in_T, out_T, W_T, 27392, narrow) \
125 + f(in_T, out_T, W_T, 27648, narrow) \
126 f(in_T, out_T, W_T, 28672, narrow) \
127 f(in_T, out_T, W_T, 32000, narrow) \
128 f(in_T, out_T, W_T, 32256, narrow) \
tests/lora/test_punica.py
+1
@@ -79,6 +79,7 @@ H1 = H2 = [
79 22016,
80 24576,
81 27392,
82 + 27648,
83 32000,
84 32256,
85 32512,