@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a53222544c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Kernel] Add punica dimension for Swallow-MS-7B LoRA (#4134)

Shoichi Uchinami committed Apr 18, 2024 at 02:02 UTC a53222544c6385ee314a26fdf42eb14f5b4e5ad9
2 files changed +2
csrc/punica/bgmv/bgmv_config.h
+1
@@ -60,6 +60,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
60 f(in_T, out_T, W_T, narrow, 32768) \
61 f(in_T, out_T, W_T, narrow, 33024) \
62 f(in_T, out_T, W_T, narrow, 36864) \
63 + f(in_T, out_T, W_T, narrow, 43264) \
64 f(in_T, out_T, W_T, narrow, 49152) \
65 f(in_T, out_T, W_T, narrow, 64000) \
66 f(in_T, out_T, W_T, narrow, 64256) \
tests/lora/test_punica.py
+1
@@ -82,6 +82,7 @@ H1 = H2 = [
82 32768,
83 33024,
84 36864,
85 + 43264,
86 49152,
87 64000,
88 64256,