A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Kernel] Add punica dimension for Baichuan-13B (#4053)
Jee Li committed
Apr 13, 2024 at 22:55 UTC
989ae2538df211ca3a31f77ac8e106c5c97c6e53
3 files changed
+3
-1
csrc/punica/bgmv/bgmv_config.h
+1
@@ -47,6 +47,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
47
f(in_T, out_T, W_T, narrow, 13696) \
48
f(in_T, out_T, W_T, narrow, 13824) \
49
f(in_T, out_T, W_T, narrow, 14336) \
50
+ f(in_T, out_T, W_T, narrow, 15360) \
51
f(in_T, out_T, W_T, narrow, 16384) \
52
f(in_T, out_T, W_T, narrow, 20480) \
53
f(in_T, out_T, W_T, narrow, 22016) \
tests/lora/test_baichuan.py
+1
-1
@@ -62,7 +62,7 @@ def test_baichuan_lora(baichuan_lora_files):
62
63
64
@pytest.mark.skip("Requires multiple GPUs")
65
-def test_llama_tensor_parallel_equality(baichuan_lora_files):
65
+def test_baichuan_tensor_parallel_equality(baichuan_lora_files):
66
# Cannot use as it will initialize torch.cuda too early...
67
# if torch.cuda.device_count() < 4:
68
# pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
tests/lora/test_punica.py
+1
@@ -72,6 +72,7 @@ H1 = H2 = [
72
11008,
73
13824,
74
14336,
75
+ 15360,
76
22016,
77
24576,
78
27392,