@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 989ae2538d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Kernel] Add punica dimension for Baichuan-13B (#4053)

Jee Li committed Apr 13, 2024 at 22:55 UTC 989ae2538df211ca3a31f77ac8e106c5c97c6e53
3 files changed +3 -1
csrc/punica/bgmv/bgmv_config.h
+1
@@ -47,6 +47,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
47 f(in_T, out_T, W_T, narrow, 13696) \
48 f(in_T, out_T, W_T, narrow, 13824) \
49 f(in_T, out_T, W_T, narrow, 14336) \
50 + f(in_T, out_T, W_T, narrow, 15360) \
51 f(in_T, out_T, W_T, narrow, 16384) \
52 f(in_T, out_T, W_T, narrow, 20480) \
53 f(in_T, out_T, W_T, narrow, 22016) \
tests/lora/test_baichuan.py
+1 -1
@@ -62,7 +62,7 @@ def test_baichuan_lora(baichuan_lora_files):
62
63
64 @pytest.mark.skip("Requires multiple GPUs")
65 -def test_llama_tensor_parallel_equality(baichuan_lora_files):
65 +def test_baichuan_tensor_parallel_equality(baichuan_lora_files):
66 # Cannot use as it will initialize torch.cuda too early...
67 # if torch.cuda.device_count() < 4:
68 # pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
tests/lora/test_punica.py
+1
@@ -72,6 +72,7 @@ H1 = H2 = [
72 11008,
73 13824,
74 14336,
75 + 15360,
76 22016,
77 24576,
78 27392,